mlx-bun 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -93
- package/docs/dag/training-inference-map.html +575 -0
- package/docs/design/adapters-end-to-end.md +50 -46
- package/docs/design/compat-cli-surface-design.md +10 -8
- package/docs/design/docs-reorg-plan.md +14 -1
- package/docs/design/hlg-sampling.md +12 -2
- package/docs/design/memory-system.md +193 -295
- package/docs/design/optimization_plan.md +4 -2
- package/docs/design/orpo-dynamic-lambda.md +179 -0
- package/docs/design/orpo-future-enhancements.md +340 -0
- package/docs/design/orpo-training.md +1212 -0
- package/docs/design/parallel-slots.md +34 -26
- package/docs/design/parity-tier-dag.md +143 -0
- package/docs/design/runtime-isolation.md +1 -1
- package/docs/design/segmented-backward-training.md +58 -39
- package/docs/design/tauri-desktop-app.md +127 -0
- package/docs/design/w4a16-compute-precision-spike.md +232 -0
- package/docs/investigations/orpo-base-uf-experiment-and-directions.md +120 -0
- package/docs/investigations/orpo-flash-cce-pin-leak.md +167 -0
- package/docs/investigations/orpo-uf-testing-handoff.md +126 -0
- package/docs/{segmented-backward-handoff.md → investigations/segmented-backward-handoff.md} +1 -1
- package/docs/investigations/starter-model-port-handoff.md +32 -13
- package/docs/investigations/steel-flash-cce-handoff.md +489 -0
- package/docs/investigations/trainer-validation-experiment.md +173 -0
- package/docs/planning/IDEAS.md +29 -0
- package/docs/planning/PRODUCT_ROADMAP.md +147 -27
- package/docs/planning/chunk-task-roadmap.md +1 -1
- package/docs/planning/journal.md +161 -0
- package/docs/reference/distribution.md +61 -5
- package/docs/reference/library-api.md +29 -14
- package/docs/reference/memory.md +77 -0
- package/docs/reference/orpo-quickstart.md +130 -0
- package/docs/reference/server-api.md +105 -7
- package/docs/reference/server-config.md +12 -5
- package/docs/reference/training.md +296 -0
- package/package.json +1 -1
- package/src/cli.ts +651 -18
- package/src/eval/ifeval.ts +138 -0
- package/src/eval/runner.ts +76 -6
- package/src/eval/tasks/bfcl.ts +49 -15
- package/src/eval/tasks/gsm8k.ts +21 -9
- package/src/eval/tasks/hashhop.ts +33 -4
- package/src/eval/tasks/humaneval.ts +30 -8
- package/src/eval/tasks/ifeval.ts +52 -18
- package/src/eval/tasks/mmlu.ts +85 -74
- package/src/eval/ultrafeedback.ts +60 -0
- package/src/jobs/runner.ts +6 -0
- package/src/lora.ts +12 -7
- package/src/memory/chunk.ts +25 -0
- package/src/memory/cluster.ts +24 -0
- package/src/memory/db.ts +114 -0
- package/src/memory/ingest.ts +22 -0
- package/src/memory/pipeline.ts +63 -0
- package/src/memory/prompts.ts +23 -0
- package/src/memory/schedule.ts +163 -0
- package/src/memory/synthesize.ts +29 -0
- package/src/memory/tools.ts +399 -0
- package/src/memory/vault.ts +694 -0
- package/src/memory/wikify.ts +23 -0
- package/src/mlx/metal-kernel.ts +31 -4
- package/src/model/flash-attention.ts +37 -19
- package/src/model/fused-geglu-kernel.ts +111 -0
- package/src/model/gemma4-base.ts +39 -1
- package/src/model/gemma4.ts +196 -21
- package/src/model/minicpm5.ts +44 -3
- package/src/pi-provider.ts +29 -13
- package/src/pi-session.ts +81 -0
- package/src/pi-terminal.ts +13 -24
- package/src/pi-web.ts +246 -166
- package/src/quantize/job.ts +17 -2
- package/src/quantize/quantizer.ts +26 -2
- package/src/server.ts +216 -47
- package/src/tokenizer.ts +6 -2
- package/src/train/dataset.ts +29 -6
- package/src/train/flash-cce.ts +1175 -0
- package/src/train/job.ts +64 -3
- package/src/train/lora-params.ts +53 -4
- package/src/train/loss.ts +947 -0
- package/src/train/optimizer.ts +10 -3
- package/src/train/prefix-shared.ts +435 -0
- package/src/train/rank.ts +36 -0
- package/src/train/segmented.ts +1148 -3
- package/src/train/steel-qmm-header.ts +1461 -0
- package/src/train/trainer.ts +570 -54
- package/src/train/watch.ts +542 -0
- package/src/web/app.html +285 -19
- package/src/web/skills/memory/SKILL.md +97 -0
- package/src/web/skills/web-research/SKILL.md +3 -1
- package/src/web/skills.ts +46 -8
- /package/{docs → src/assets}/curve-designer.html +0 -0
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
// `mlx-bun train-watch <adapter-dir>` — a mactop-style live dashboard for a
|
|
2
|
+
// training run. It tails the trainer's append-only `<adapter>/metrics.jsonl`
|
|
3
|
+
// (written by trainLora) and renders auto-scaled line charts for loss, margin,
|
|
4
|
+
// accuracy (train line + val markers) and memory, plus a progress/ETA header.
|
|
5
|
+
//
|
|
6
|
+
// Decoupled by design: the trainer only writes the stream, this viewer only
|
|
7
|
+
// reads it. So it attaches/detaches freely to a LIVE run, replays a FINISHED
|
|
8
|
+
// one, and never touches training. The renderer is a PURE function
|
|
9
|
+
// (renderFrame) over parsed state, so it can be exercised against a captured
|
|
10
|
+
// fixture with no TTY.
|
|
11
|
+
|
|
12
|
+
import { readFileSync } from "node:fs";
|
|
13
|
+
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// Parsed run state
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
export interface Pt { step: number; t: number; y: number }
|
|
19
|
+
/** A validation point — sparse (one per eval step). Accuracy carries its raw
|
|
20
|
+
* correct/total so it can be shown as a count, not a bare ratio. */
|
|
21
|
+
export interface ValPt extends Pt { nc?: number; nt?: number }
|
|
22
|
+
export interface RunState {
|
|
23
|
+
meta: {
|
|
24
|
+
method: string; model: string; iters: number; lambda?: number;
|
|
25
|
+
lr?: number; seq?: number; startedAt?: number; adapterPath?: string;
|
|
26
|
+
};
|
|
27
|
+
// Dense per-iter series (a line per metric).
|
|
28
|
+
loss: Pt[];
|
|
29
|
+
margin: Pt[];
|
|
30
|
+
trainAcc: Pt[]; // y ∈ {0,1} at B=1 — render as a rolling fraction, not raw
|
|
31
|
+
mem: Pt[]; // y = active GB (resident — flat = no leak)
|
|
32
|
+
peak: Pt[]; // y = peak GB (cumulative high-water — the climb-to-plateau)
|
|
33
|
+
// Sparse per-eval series (markers, not lines — they only exist at eval steps).
|
|
34
|
+
valAcc: ValPt[];
|
|
35
|
+
valLoss: Pt[];
|
|
36
|
+
peakGb: number; // running high-water
|
|
37
|
+
lastStep: number;
|
|
38
|
+
lastT: number;
|
|
39
|
+
done: boolean;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function emptyState(): RunState {
|
|
43
|
+
return {
|
|
44
|
+
meta: { method: "?", model: "", iters: 0 },
|
|
45
|
+
loss: [], margin: [], trainAcc: [], mem: [], peak: [],
|
|
46
|
+
valAcc: [], valLoss: [],
|
|
47
|
+
peakGb: 0, lastStep: 0, lastT: 0, done: false,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Fold one parsed JSONL record into the state. */
|
|
52
|
+
function ingest(s: RunState, o: Record<string, unknown>): void {
|
|
53
|
+
const type = o.type;
|
|
54
|
+
if (type === "meta") {
|
|
55
|
+
s.meta = {
|
|
56
|
+
method: String(o.method ?? "?"),
|
|
57
|
+
model: String(o.model ?? ""),
|
|
58
|
+
iters: Number(o.iters ?? 0),
|
|
59
|
+
lambda: o.orpo_lambda != null ? Number(o.orpo_lambda) : undefined,
|
|
60
|
+
lr: o.learning_rate != null ? Number(o.learning_rate) : undefined,
|
|
61
|
+
seq: o.max_seq_length != null ? Number(o.max_seq_length) : undefined,
|
|
62
|
+
startedAt: o.t != null ? Number(o.t) : undefined,
|
|
63
|
+
adapterPath: o.adapter_path != null ? String(o.adapter_path) : undefined,
|
|
64
|
+
};
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
if (type !== "metric") return;
|
|
68
|
+
const step = Number(o.step), t = Number(o.t);
|
|
69
|
+
if (!Number.isFinite(step)) return;
|
|
70
|
+
if (o.kind === "train") {
|
|
71
|
+
if (Number.isFinite(Number(o.loss))) s.loss.push({ step, t, y: Number(o.loss) });
|
|
72
|
+
if (Number.isFinite(Number(o.margin))) s.margin.push({ step, t, y: Number(o.margin) });
|
|
73
|
+
if (Number.isFinite(Number(o.accuracy))) s.trainAcc.push({ step, t, y: Number(o.accuracy) });
|
|
74
|
+
if (Number.isFinite(Number(o.active_gb))) s.mem.push({ step, t, y: Number(o.active_gb) });
|
|
75
|
+
if (Number.isFinite(Number(o.peak_gb))) {
|
|
76
|
+
s.peak.push({ step, t, y: Number(o.peak_gb) });
|
|
77
|
+
s.peakGb = Math.max(s.peakGb, Number(o.peak_gb));
|
|
78
|
+
}
|
|
79
|
+
s.lastStep = Math.max(s.lastStep, step);
|
|
80
|
+
if (Number.isFinite(t)) s.lastT = Math.max(s.lastT, t);
|
|
81
|
+
} else if (o.kind === "val") {
|
|
82
|
+
if (Number.isFinite(Number(o.accuracy)))
|
|
83
|
+
s.valAcc.push({ step, t, y: Number(o.accuracy),
|
|
84
|
+
nc: o.n_correct != null ? Number(o.n_correct) : undefined,
|
|
85
|
+
nt: o.n_total != null ? Number(o.n_total) : undefined });
|
|
86
|
+
if (Number.isFinite(Number(o.loss))) s.valLoss.push({ step, t, y: Number(o.loss) });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Build full state from a metrics.jsonl text blob (replay-from-start). */
|
|
91
|
+
export function parseStream(text: string): RunState {
|
|
92
|
+
const s = emptyState();
|
|
93
|
+
for (const line of text.split("\n")) {
|
|
94
|
+
const t = line.trim();
|
|
95
|
+
if (!t) continue;
|
|
96
|
+
try { ingest(s, JSON.parse(t) as Record<string, unknown>); } catch { /* skip partial/garbled */ }
|
|
97
|
+
}
|
|
98
|
+
if (s.meta.iters > 0 && s.lastStep >= s.meta.iters) s.done = true;
|
|
99
|
+
return s;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Median step time (seconds) from the last K train timestamps. */
|
|
103
|
+
export function sPerStep(s: RunState, k = 30): number {
|
|
104
|
+
const pts = s.loss.slice(-k - 1);
|
|
105
|
+
const d: number[] = [];
|
|
106
|
+
for (let i = 1; i < pts.length; i++) {
|
|
107
|
+
const dt = (pts[i]!.t - pts[i - 1]!.t) / 1000;
|
|
108
|
+
if (dt > 0 && dt < 600) d.push(dt);
|
|
109
|
+
}
|
|
110
|
+
if (d.length === 0) return 0;
|
|
111
|
+
d.sort((a, b) => a - b);
|
|
112
|
+
return d[Math.floor(d.length / 2)]!;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// ---------------------------------------------------------------------------
|
|
116
|
+
// Cell grid compositor — the whole frame is built into a typed grid, then
|
|
117
|
+
// run-length colored on render. Unifies header / charts / axis labels / markers
|
|
118
|
+
// and sidesteps any mid-ANSI string splicing.
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
const DEFAULT_FG = -1;
|
|
122
|
+
interface Cell { ch: string; fg: number }
|
|
123
|
+
|
|
124
|
+
// 256-color palette (xterm).
|
|
125
|
+
const C = {
|
|
126
|
+
dim: 240, axis: 244, label: 250, title: 255,
|
|
127
|
+
loss: 203, margin: 78, accT: 45, accV: 207, mem: 221, ok: 78, warn: 203, accent: 81,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
class Grid {
|
|
131
|
+
rows: number; cols: number; cells: Cell[];
|
|
132
|
+
constructor(rows: number, cols: number) {
|
|
133
|
+
this.rows = rows; this.cols = cols;
|
|
134
|
+
this.cells = new Array(rows * cols);
|
|
135
|
+
for (let i = 0; i < this.cells.length; i++) this.cells[i] = { ch: " ", fg: DEFAULT_FG };
|
|
136
|
+
}
|
|
137
|
+
put(r: number, c: number, ch: string, fg = DEFAULT_FG): void {
|
|
138
|
+
if (r < 0 || r >= this.rows || c < 0 || c >= this.cols) return;
|
|
139
|
+
const cell = this.cells[r * this.cols + c]!;
|
|
140
|
+
cell.ch = ch; cell.fg = fg;
|
|
141
|
+
}
|
|
142
|
+
text(r: number, c: number, str: string, fg = DEFAULT_FG): void {
|
|
143
|
+
for (let i = 0; i < str.length; i++) this.put(r, c + i, str[i]!, fg);
|
|
144
|
+
}
|
|
145
|
+
/** Right-align str ending at column `cEnd`. */
|
|
146
|
+
textRight(r: number, cEnd: number, str: string, fg = DEFAULT_FG): void {
|
|
147
|
+
this.text(r, cEnd - str.length + 1, str, fg);
|
|
148
|
+
}
|
|
149
|
+
render(): string[] {
|
|
150
|
+
const out: string[] = [];
|
|
151
|
+
for (let r = 0; r < this.rows; r++) {
|
|
152
|
+
let line = "", curFg = DEFAULT_FG;
|
|
153
|
+
for (let c = 0; c < this.cols; c++) {
|
|
154
|
+
const cell = this.cells[r * this.cols + c]!;
|
|
155
|
+
if (cell.fg !== curFg) {
|
|
156
|
+
line += cell.fg === DEFAULT_FG ? "\x1b[0m" : `\x1b[38;5;${cell.fg}m`;
|
|
157
|
+
curFg = cell.fg;
|
|
158
|
+
}
|
|
159
|
+
line += cell.ch;
|
|
160
|
+
}
|
|
161
|
+
if (curFg !== DEFAULT_FG) line += "\x1b[0m";
|
|
162
|
+
out.push(line);
|
|
163
|
+
}
|
|
164
|
+
return out;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
// Braille (2×4 dot) canvas — sub-cell line resolution for the charts.
|
|
170
|
+
// ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
// dot bit for (col∈{0,1}, row∈{0..3}) within a cell.
|
|
173
|
+
const DOT = [
|
|
174
|
+
[0x01, 0x08],
|
|
175
|
+
[0x02, 0x10],
|
|
176
|
+
[0x04, 0x20],
|
|
177
|
+
[0x40, 0x80],
|
|
178
|
+
];
|
|
179
|
+
|
|
180
|
+
class Braille {
|
|
181
|
+
cols: number; rows: number; data: Uint8Array;
|
|
182
|
+
constructor(cols: number, rows: number) {
|
|
183
|
+
this.cols = cols; this.rows = rows;
|
|
184
|
+
this.data = new Uint8Array(cols * rows);
|
|
185
|
+
}
|
|
186
|
+
setDot(dx: number, dy: number): void {
|
|
187
|
+
const cx = dx >> 1, cy = dy >> 2;
|
|
188
|
+
if (cx < 0 || cx >= this.cols || cy < 0 || cy >= this.rows) return;
|
|
189
|
+
this.data[cy * this.cols + cx]! |= DOT[dy & 3]![dx & 1]!;
|
|
190
|
+
}
|
|
191
|
+
line(x0: number, y0: number, x1: number, y1: number): void {
|
|
192
|
+
let dx = Math.abs(x1 - x0), dy = -Math.abs(y1 - y0);
|
|
193
|
+
const sx = x0 < x1 ? 1 : -1, sy = y0 < y1 ? 1 : -1;
|
|
194
|
+
let err = dx + dy;
|
|
195
|
+
for (;;) {
|
|
196
|
+
this.setDot(x0, y0);
|
|
197
|
+
if (x0 === x1 && y0 === y1) break;
|
|
198
|
+
const e2 = 2 * err;
|
|
199
|
+
if (e2 >= dy) { err += dy; x0 += sx; }
|
|
200
|
+
if (e2 <= dx) { err += dx; y0 += sy; }
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
char(cx: number, cy: number): string | null {
|
|
204
|
+
const b = this.data[cy * this.cols + cx]!;
|
|
205
|
+
return b === 0 ? null : String.fromCodePoint(0x2800 | b);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// One chart panel: title row + y-axis labels + braille plot + x-axis labels.
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
|
|
213
|
+
interface ChartOpts {
|
|
214
|
+
title: string;
|
|
215
|
+
series: Pt[];
|
|
216
|
+
color: number;
|
|
217
|
+
fixed?: [number, number]; // fixed y-range (e.g. accuracy 0..1)
|
|
218
|
+
fmtY: (v: number) => string;
|
|
219
|
+
headline?: string; // colored value shown after the title
|
|
220
|
+
alert?: string; // warn-colored note after the headline (e.g. "▲ rising")
|
|
221
|
+
zeroLine?: boolean; // dashed y=0 reference
|
|
222
|
+
refLine?: number; // dashed reference rule at an arbitrary y (e.g. 0.5 chance)
|
|
223
|
+
markers?: { pts: Pt[]; color: number }; // discrete sparse overlay (val points)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function fmtNum(v: number): string {
|
|
227
|
+
if (!Number.isFinite(v)) return "—";
|
|
228
|
+
const a = Math.abs(v);
|
|
229
|
+
if (a >= 100) return v.toFixed(0);
|
|
230
|
+
if (a >= 10) return v.toFixed(1);
|
|
231
|
+
if (a >= 1) return v.toFixed(2);
|
|
232
|
+
return v.toFixed(3);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function compactInt(n: number): string {
|
|
236
|
+
if (n >= 1_000_000) return (n / 1_000_000).toFixed(n % 1_000_000 ? 1 : 0) + "M";
|
|
237
|
+
if (n >= 10_000) return (n / 1000).toFixed(0) + "k";
|
|
238
|
+
return String(n);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** Draw a chart into `grid` at top-left (r0,c0) spanning (h rows, w cols). */
|
|
242
|
+
function drawChart(grid: Grid, r0: number, c0: number, h: number, w: number, o: ChartOpts): void {
|
|
243
|
+
// Title line: title · headline (series color) · alert (warn).
|
|
244
|
+
grid.text(r0, c0, o.title, C.title);
|
|
245
|
+
let hx = c0 + o.title.length + 1;
|
|
246
|
+
if (o.headline) { grid.text(r0, hx, o.headline, o.color); hx += o.headline.length + 2; }
|
|
247
|
+
if (o.alert) grid.text(r0, hx, o.alert, C.warn);
|
|
248
|
+
|
|
249
|
+
const plotTop = r0 + 1;
|
|
250
|
+
const plotRows = h - 2; // minus title + x-axis row
|
|
251
|
+
if (plotRows < 1 || w < 12) return;
|
|
252
|
+
|
|
253
|
+
const pts = o.series;
|
|
254
|
+
// y-range.
|
|
255
|
+
let ymin: number, ymax: number;
|
|
256
|
+
if (o.fixed) { [ymin, ymax] = o.fixed; }
|
|
257
|
+
else if (pts.length === 0) { ymin = 0; ymax = 1; }
|
|
258
|
+
else {
|
|
259
|
+
ymin = Infinity; ymax = -Infinity;
|
|
260
|
+
for (const p of pts) { if (p.y < ymin) ymin = p.y; if (p.y > ymax) ymax = p.y; }
|
|
261
|
+
if (o.markers) for (const p of o.markers.pts) { if (p.y < ymin) ymin = p.y; if (p.y > ymax) ymax = p.y; }
|
|
262
|
+
if (o.zeroLine) { ymin = Math.min(ymin, 0); ymax = Math.max(ymax, 0); }
|
|
263
|
+
if (ymin === ymax) { ymin -= 1; ymax += 1; }
|
|
264
|
+
else { const pad = (ymax - ymin) * 0.08; ymin -= pad; ymax += pad; }
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// gutter for y labels.
|
|
268
|
+
const gutter = Math.max(o.fmtY(ymin).length, o.fmtY(ymax).length) + 1;
|
|
269
|
+
const plotCols = w - gutter;
|
|
270
|
+
if (plotCols < 4) return;
|
|
271
|
+
|
|
272
|
+
// y-axis labels: top=max, bottom=min, mid where room.
|
|
273
|
+
grid.textRight(plotTop, c0 + gutter - 2, o.fmtY(ymax), C.axis);
|
|
274
|
+
grid.textRight(plotTop + plotRows - 1, c0 + gutter - 2, o.fmtY(ymin), C.axis);
|
|
275
|
+
if (plotRows >= 5) grid.textRight(plotTop + (plotRows >> 1), c0 + gutter - 2, o.fmtY((ymin + ymax) / 2), C.axis);
|
|
276
|
+
// vertical axis rule.
|
|
277
|
+
for (let r = 0; r < plotRows; r++) grid.put(plotTop + r, c0 + gutter - 1, "│", C.dim);
|
|
278
|
+
|
|
279
|
+
// x-range (step).
|
|
280
|
+
const xmin = pts.length ? pts[0]!.step : 0;
|
|
281
|
+
const xmax = pts.length ? pts[pts.length - 1]!.step : 1;
|
|
282
|
+
const xspan = Math.max(1, xmax - xmin);
|
|
283
|
+
|
|
284
|
+
const nDotX = plotCols * 2, nDotY = plotRows * 4;
|
|
285
|
+
const canvas = new Braille(plotCols, plotRows);
|
|
286
|
+
const mapX = (step: number) => Math.round(((step - xmin) / xspan) * (nDotX - 1));
|
|
287
|
+
const mapY = (y: number) => Math.round((1 - (y - ymin) / (ymax - ymin)) * (nDotY - 1));
|
|
288
|
+
|
|
289
|
+
// zero reference: a dim dotted grid rule the data line crosses (drawn first,
|
|
290
|
+
// so the braille blit below overwrites it only where there's data).
|
|
291
|
+
const drawRule = (yval: number) => {
|
|
292
|
+
if (!(yval > ymin && yval < ymax)) return;
|
|
293
|
+
const rRow = plotTop + Math.round((1 - (yval - ymin) / (ymax - ymin)) * (plotRows - 1));
|
|
294
|
+
for (let c = 0; c < plotCols; c++) grid.put(rRow, c0 + gutter + c, "┄", C.dim);
|
|
295
|
+
};
|
|
296
|
+
if (o.zeroLine) drawRule(0);
|
|
297
|
+
if (o.refLine != null) drawRule(o.refLine);
|
|
298
|
+
|
|
299
|
+
// Downsample to keep line cost bounded on long runs.
|
|
300
|
+
if (pts.length > 0) {
|
|
301
|
+
const stride = Math.max(1, Math.floor(pts.length / (nDotX * 2)));
|
|
302
|
+
let prevX = mapX(pts[0]!.step), prevY = mapY(pts[0]!.y);
|
|
303
|
+
canvas.setDot(prevX, prevY);
|
|
304
|
+
for (let i = stride; i < pts.length; i += stride) {
|
|
305
|
+
const cx = mapX(pts[i]!.step), cy = mapY(pts[i]!.y);
|
|
306
|
+
canvas.line(prevX, prevY, cx, cy);
|
|
307
|
+
prevX = cx; prevY = cy;
|
|
308
|
+
}
|
|
309
|
+
// always connect the final point.
|
|
310
|
+
const last = pts[pts.length - 1]!;
|
|
311
|
+
canvas.line(prevX, prevY, mapX(last.step), mapY(last.y));
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// Blit braille into grid.
|
|
315
|
+
for (let r = 0; r < plotRows; r++)
|
|
316
|
+
for (let c = 0; c < plotCols; c++) {
|
|
317
|
+
const ch = canvas.char(c, r);
|
|
318
|
+
if (ch) grid.put(plotTop + r, c0 + gutter + c, ch, o.color);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// Discrete markers (val accuracy) overlaid as ●.
|
|
322
|
+
if (o.markers) {
|
|
323
|
+
for (const p of o.markers.pts) {
|
|
324
|
+
const cx = c0 + gutter + Math.round(((p.step - xmin) / xspan) * (plotCols - 1));
|
|
325
|
+
const cy = plotTop + Math.round((1 - (p.y - ymin) / (ymax - ymin)) * (plotRows - 1));
|
|
326
|
+
grid.put(cy, cx, "●", o.markers.color);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// x-axis labels row.
|
|
331
|
+
const axisRow = plotTop + plotRows;
|
|
332
|
+
grid.text(axisRow, c0 + gutter, compactInt(xmin), C.axis);
|
|
333
|
+
grid.textRight(axisRow, c0 + gutter + plotCols - 1, compactInt(xmax), C.axis);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// ---------------------------------------------------------------------------
|
|
337
|
+
// Full-frame renderer (pure).
|
|
338
|
+
// ---------------------------------------------------------------------------
|
|
339
|
+
|
|
340
|
+
/** Rolling MEDIAN over a trailing window — robust trend for a metric whose
|
|
341
|
+
* per-step value is per-example (B=1 loss spikes on a long/hard pair are
|
|
342
|
+
* rejected, unlike a mean). This is what makes the loss trend legible. */
|
|
343
|
+
function rollingMedian(pts: Pt[], w: number): Pt[] {
|
|
344
|
+
const out: Pt[] = [];
|
|
345
|
+
for (let i = 0; i < pts.length; i++) {
|
|
346
|
+
const lo = Math.max(0, i - w + 1);
|
|
347
|
+
const win: number[] = [];
|
|
348
|
+
for (let j = lo; j <= i; j++) win.push(pts[j]!.y);
|
|
349
|
+
win.sort((a, b) => a - b);
|
|
350
|
+
out.push({ step: pts[i]!.step, t: pts[i]!.t, y: win[win.length >> 1]! });
|
|
351
|
+
}
|
|
352
|
+
return out;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/** Light trailing mean — smooths the piecewise-constant plateaus a rolling
|
|
356
|
+
* median leaves behind (median despikes; this mean removes the "banding" those
|
|
357
|
+
* flat runs make in the braille line). */
|
|
358
|
+
function rollingMean(pts: Pt[], w: number): Pt[] {
|
|
359
|
+
const out: Pt[] = [];
|
|
360
|
+
let sum = 0;
|
|
361
|
+
for (let i = 0; i < pts.length; i++) {
|
|
362
|
+
sum += pts[i]!.y;
|
|
363
|
+
if (i >= w) sum -= pts[i - w]!.y;
|
|
364
|
+
out.push({ step: pts[i]!.step, t: pts[i]!.t, y: sum / Math.min(i + 1, w) });
|
|
365
|
+
}
|
|
366
|
+
return out;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function minPt(pts: Pt[]): { y: number; step: number } | null {
|
|
370
|
+
if (!pts.length) return null;
|
|
371
|
+
let y = Infinity, step = pts[0]!.step;
|
|
372
|
+
for (const p of pts) if (p.y < y) { y = p.y; step = p.step; }
|
|
373
|
+
return { y, step };
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Pull a readable model name from an HF cache path
|
|
377
|
+
* (".../models--mlx-community--MiniCPM5-1B-OptiQ-4bit/snapshots/<hash>") or a
|
|
378
|
+
* plain repo id, dropping the quant suffix. */
|
|
379
|
+
function shortModel(p: string): string {
|
|
380
|
+
const m = p.match(/models--[^/]*--([^/]+)/);
|
|
381
|
+
const name = m ? m[1]! : (p.split("/").filter((s) => s && s !== "snapshots" && !/^[0-9a-f]{16,}$/.test(s)).pop() ?? p);
|
|
382
|
+
return name.replace(/-OptiQ-4bit$/i, "");
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function fmtDuration(sec: number): string {
|
|
386
|
+
if (!Number.isFinite(sec) || sec < 0) return "—";
|
|
387
|
+
const h = Math.floor(sec / 3600), m = Math.floor((sec % 3600) / 60), s = Math.floor(sec % 60);
|
|
388
|
+
if (h > 0) return `${h}h${String(m).padStart(2, "0")}m`;
|
|
389
|
+
if (m > 0) return `${m}m${String(s).padStart(2, "0")}s`;
|
|
390
|
+
return `${s}s`;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export function renderFrame(st: RunState, cols: number, rows: number): string[] {
|
|
394
|
+
const grid = new Grid(rows, cols);
|
|
395
|
+
const sps = sPerStep(st);
|
|
396
|
+
const iters = st.meta.iters || st.lastStep || 1;
|
|
397
|
+
const frac = Math.min(1, st.lastStep / iters);
|
|
398
|
+
const eta = sps > 0 ? (iters - st.lastStep) * sps : NaN;
|
|
399
|
+
const elapsed = st.meta.startedAt && st.lastT ? (st.lastT - st.meta.startedAt) / 1000 : NaN;
|
|
400
|
+
|
|
401
|
+
// Header line 1: identity + progress.
|
|
402
|
+
const modelShort = shortModel(st.meta.model);
|
|
403
|
+
const statusDot = st.done ? "✓" : "●";
|
|
404
|
+
const statusFg = st.done ? C.ok : C.accent;
|
|
405
|
+
grid.put(0, 0, statusDot, statusFg);
|
|
406
|
+
grid.text(0, 2, `${st.meta.method} · ${modelShort}`, C.title);
|
|
407
|
+
const hdr = st.done
|
|
408
|
+
? `step ${st.lastStep}/${iters} · complete · ${fmtDuration(elapsed)}`
|
|
409
|
+
: `step ${st.lastStep}/${iters} · ${(frac * 100).toFixed(1)}% · ETA ${fmtDuration(eta)} · ${sps ? sps.toFixed(2) + "s/step" : "—"} · ${fmtDuration(elapsed)}`;
|
|
410
|
+
grid.textRight(0, cols - 1, hdr, C.label);
|
|
411
|
+
|
|
412
|
+
// Progress bar (row 1).
|
|
413
|
+
const barW = cols - 2;
|
|
414
|
+
const filled = Math.round(frac * barW);
|
|
415
|
+
for (let i = 0; i < barW; i++) grid.put(1, 1 + i, i < filled ? "█" : "░", i < filled ? statusFg : C.dim);
|
|
416
|
+
|
|
417
|
+
// Legend (row 2): trend lines are smoothed because B=1 per-step is per-example;
|
|
418
|
+
// ● markers are val (sparse, one per eval step).
|
|
419
|
+
grid.text(2, 1, "loss = orpo/step · margin = confidence · accuracy = correct on val · ● = val @ eval", C.dim);
|
|
420
|
+
|
|
421
|
+
// Chart stack (rows 2..).
|
|
422
|
+
const top = 3;
|
|
423
|
+
const avail = rows - top;
|
|
424
|
+
if (avail < 6) return grid.render();
|
|
425
|
+
|
|
426
|
+
const cur = (a: Pt[]) => (a.length ? a[a.length - 1]!.y : NaN);
|
|
427
|
+
const pct = (v: number) => (Number.isFinite(v) ? `${(v * 100).toFixed(1)}%` : "—");
|
|
428
|
+
const SMOOTH = 15;
|
|
429
|
+
|
|
430
|
+
// Loss: at B=1 the per-step loss is per-EXAMPLE (a long/hard pair spikes it),
|
|
431
|
+
// so the displayed trend is a rolling MEDIAN (robust to those spikes) plus the
|
|
432
|
+
// sparse val-loss markers. The min/rising detector runs on the median — the
|
|
433
|
+
// "loss bottomed out and turned back up" early-stop signal.
|
|
434
|
+
const lossMed = rollingMean(rollingMedian(st.loss, SMOOTH), 5); // median despikes, mean de-bands
|
|
435
|
+
const lm = minPt(lossMed);
|
|
436
|
+
const recentLoss = cur(lossMed);
|
|
437
|
+
const rising = !!(lm && lossMed.length > SMOOTH + 5 && lm.step < st.lastStep - SMOOTH
|
|
438
|
+
&& recentLoss > lm.y * 1.05);
|
|
439
|
+
|
|
440
|
+
const marginMed = rollingMean(rollingMedian(st.margin, SMOOTH), 5); // median despikes, mean de-bands
|
|
441
|
+
|
|
442
|
+
// Val accuracy — the metric that matters: of the held-out val set, how many
|
|
443
|
+
// pairs it correctly prefers (chosen > rejected). Sparse (one per eval step),
|
|
444
|
+
// drawn as a line + markers; 50% is chance.
|
|
445
|
+
const lastVal = st.valAcc[st.valAcc.length - 1];
|
|
446
|
+
const bestVal = st.valAcc.reduce<ValPt | null>((b, p) => (!b || p.y > b.y ? p : b), null);
|
|
447
|
+
|
|
448
|
+
const panels: ChartOpts[] = [
|
|
449
|
+
{
|
|
450
|
+
title: "loss · orpo per step", series: lossMed, color: C.loss, fmtY: fmtNum,
|
|
451
|
+
markers: st.valLoss.length ? { pts: st.valLoss, color: C.accV } : undefined,
|
|
452
|
+
headline: `${fmtNum(recentLoss)}${lm ? ` ↓min ${fmtNum(lm.y)} @ ${lm.step}` : ""}`,
|
|
453
|
+
alert: rising ? `▲ rising +${fmtNum(recentLoss - lm!.y)} since ${lm!.step}` : undefined,
|
|
454
|
+
},
|
|
455
|
+
{
|
|
456
|
+
title: "margin · confidence in the choice", series: marginMed, color: C.margin,
|
|
457
|
+
fmtY: (v) => (v >= 0 ? "+" : "") + fmtNum(v),
|
|
458
|
+
zeroLine: true,
|
|
459
|
+
headline: `${cur(marginMed) >= 0 ? "+" : ""}${fmtNum(cur(marginMed))} ${cur(marginMed) > 0 ? "leans chosen" : "leans rejected"}`,
|
|
460
|
+
},
|
|
461
|
+
{
|
|
462
|
+
title: "accuracy · correct choices on val", series: st.valAcc, color: C.accT, fixed: [0, 1],
|
|
463
|
+
fmtY: (v) => `${Math.round(v * 100)}%`,
|
|
464
|
+
refLine: 0.5,
|
|
465
|
+
markers: { pts: st.valAcc, color: C.accT },
|
|
466
|
+
headline: lastVal
|
|
467
|
+
? `${lastVal.nc ?? "?"}/${lastVal.nt ?? "?"} correct = ${pct(lastVal.y)}`
|
|
468
|
+
+ (bestVal ? ` · best ${pct(bestVal.y)} @ ${bestVal.step}` : "")
|
|
469
|
+
: "awaiting first eval (50% = chance)",
|
|
470
|
+
},
|
|
471
|
+
{
|
|
472
|
+
title: "memory", series: st.peak, color: C.mem, fmtY: (v) => v.toFixed(1) + "G",
|
|
473
|
+
headline: `peak ${st.peakGb.toFixed(2)}GB · active ${fmtNum(cur(st.mem))}GB`,
|
|
474
|
+
},
|
|
475
|
+
];
|
|
476
|
+
|
|
477
|
+
const ph = Math.floor(avail / panels.length);
|
|
478
|
+
if (ph < 4) {
|
|
479
|
+
// Too short: only loss.
|
|
480
|
+
drawChart(grid, top, 1, avail, cols - 2, panels[0]!);
|
|
481
|
+
return grid.render();
|
|
482
|
+
}
|
|
483
|
+
panels.forEach((p, i) => drawChart(grid, top + i * ph, 1, ph, cols - 2, p));
|
|
484
|
+
return grid.render();
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// ---------------------------------------------------------------------------
|
|
488
|
+
// Live TUI loop.
|
|
489
|
+
// ---------------------------------------------------------------------------
|
|
490
|
+
|
|
491
|
+
const ALT_ON = "\x1b[?1049h\x1b[?25l", ALT_OFF = "\x1b[?25h\x1b[?1049l\x1b[0m";
|
|
492
|
+
const HOME = "\x1b[H", CLEAR = "\x1b[2J";
|
|
493
|
+
|
|
494
|
+
export async function runWatch(adapterDir: string): Promise<void> {
|
|
495
|
+
const path = `${adapterDir}/metrics.jsonl`;
|
|
496
|
+
if (!(await Bun.file(path).exists())) {
|
|
497
|
+
console.error(`no metrics.jsonl in ${adapterDir} — is this an mlx-bun training run dir?`);
|
|
498
|
+
console.error(`(the trainer writes ${path} as it runs; point train-watch at the --adapter dir)`);
|
|
499
|
+
process.exit(1);
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
let lastRender = "";
|
|
503
|
+
const draw = (st: RunState) => {
|
|
504
|
+
const cols = process.stdout.columns ?? 100;
|
|
505
|
+
const rows = process.stdout.rows ?? 30;
|
|
506
|
+
const frame = HOME + renderFrame(st, cols, rows).join("\n");
|
|
507
|
+
if (frame !== lastRender) { process.stdout.write(frame); lastRender = frame; }
|
|
508
|
+
};
|
|
509
|
+
|
|
510
|
+
const restore = () => { process.stdout.write(ALT_OFF); };
|
|
511
|
+
const quit = (code = 0) => {
|
|
512
|
+
if (process.stdin.isTTY) process.stdin.setRawMode(false);
|
|
513
|
+
restore();
|
|
514
|
+
process.exit(code);
|
|
515
|
+
};
|
|
516
|
+
|
|
517
|
+
process.stdout.write(ALT_ON + CLEAR);
|
|
518
|
+
if (process.stdin.isTTY) {
|
|
519
|
+
process.stdin.setRawMode(true);
|
|
520
|
+
process.stdin.resume();
|
|
521
|
+
process.stdin.on("data", (b) => {
|
|
522
|
+
const s = b.toString();
|
|
523
|
+
if (s === "q" || s === "\x03" /* ^C */) quit(0);
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
process.on("SIGINT", () => quit(0));
|
|
527
|
+
process.on("SIGTERM", () => quit(0));
|
|
528
|
+
process.stdout.on("resize", () => { lastRender = ""; });
|
|
529
|
+
|
|
530
|
+
// Poll the file; reparse fully each tick (it's small, single-digit MB) — keeps
|
|
531
|
+
// the loop dead simple and robust to truncation/rotation. Stop redrawing once
|
|
532
|
+
// the run is done AND the file has settled.
|
|
533
|
+
let stableTicks = 0;
|
|
534
|
+
for (;;) {
|
|
535
|
+
let st: RunState;
|
|
536
|
+
try { st = parseStream(readFileSync(path, "utf8")); }
|
|
537
|
+
catch { st = emptyState(); }
|
|
538
|
+
draw(st);
|
|
539
|
+
if (st.done) { if (++stableTicks > 8) { /* keep showing final frame, slow poll */ } }
|
|
540
|
+
await Bun.sleep(st.done ? 1000 : 250);
|
|
541
|
+
}
|
|
542
|
+
}
|