@tuned-tensor/local 0.2.8 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +67 -0
  2. package/README.md +84 -204
  3. package/dist/artifacts.d.ts +3 -4
  4. package/dist/artifacts.js +55 -33
  5. package/dist/artifacts.js.map +1 -1
  6. package/dist/compare.d.ts +1 -8
  7. package/dist/compare.js +1 -23
  8. package/dist/compare.js.map +1 -1
  9. package/dist/contracts.d.ts +59 -366
  10. package/dist/contracts.js +73 -224
  11. package/dist/contracts.js.map +1 -1
  12. package/dist/dataset.d.ts +2 -4
  13. package/dist/dataset.js +31 -125
  14. package/dist/dataset.js.map +1 -1
  15. package/dist/doctor.d.ts +2 -3
  16. package/dist/doctor.js +23 -161
  17. package/dist/doctor.js.map +1 -1
  18. package/dist/evaluation.d.ts +11 -71
  19. package/dist/evaluation.js +212 -572
  20. package/dist/evaluation.js.map +1 -1
  21. package/dist/huggingface-cache.d.ts +8 -7
  22. package/dist/huggingface-cache.js +16 -8
  23. package/dist/huggingface-cache.js.map +1 -1
  24. package/dist/index.d.ts +0 -10
  25. package/dist/index.js +205 -628
  26. package/dist/index.js.map +1 -1
  27. package/dist/local-project.d.ts +1 -11
  28. package/dist/local-project.js +33 -61
  29. package/dist/local-project.js.map +1 -1
  30. package/dist/model-registry.d.ts +3 -16
  31. package/dist/model-registry.js +76 -292
  32. package/dist/model-registry.js.map +1 -1
  33. package/dist/model-server.d.ts +1 -2
  34. package/dist/model-server.js +9 -18
  35. package/dist/model-server.js.map +1 -1
  36. package/dist/orchestrator.d.ts +11 -25
  37. package/dist/orchestrator.js +246 -566
  38. package/dist/orchestrator.js.map +1 -1
  39. package/dist/prefetch.d.ts +1 -5
  40. package/dist/prefetch.js +14 -19
  41. package/dist/prefetch.js.map +1 -1
  42. package/dist/process-runner.d.ts +10 -29
  43. package/dist/process-runner.js +63 -169
  44. package/dist/process-runner.js.map +1 -1
  45. package/dist/process-training.d.ts +0 -1
  46. package/dist/process-training.js +10 -87
  47. package/dist/process-training.js.map +1 -1
  48. package/dist/store.d.ts +1 -3
  49. package/dist/store.js +92 -290
  50. package/dist/store.js.map +1 -1
  51. package/docs/architecture.md +87 -152
  52. package/docs/spark.md +66 -97
  53. package/examples/dry-runner.json +14 -0
  54. package/examples/local-runner.json +8 -6
  55. package/examples/smoke-spec.json +41 -0
  56. package/package.json +6 -6
  57. package/training/local-runner/pyproject.toml +0 -5
  58. package/training/local-runner/src/evaluate.py +89 -234
  59. package/training/local-runner/src/model_contract.py +47 -0
  60. package/training/local-runner/src/prefetch.py +18 -4
  61. package/training/local-runner/src/serve.py +54 -115
  62. package/training/local-runner/src/sft_data.py +97 -0
  63. package/training/local-runner/src/train.py +146 -346
  64. package/training/local-runner/uv.lock +0 -1197
  65. package/dist/labeling-sanitize.d.ts +0 -31
  66. package/dist/labeling-sanitize.js +0 -158
  67. package/dist/labeling-sanitize.js.map +0 -1
  68. package/dist/labeling.d.ts +0 -155
  69. package/dist/labeling.js +0 -496
  70. package/dist/labeling.js.map +0 -1
  71. package/dist/openrouter.d.ts +0 -29
  72. package/dist/openrouter.js +0 -66
  73. package/dist/openrouter.js.map +0 -1
  74. package/dist/server.d.ts +0 -9
  75. package/dist/server.js +0 -211
  76. package/dist/server.js.map +0 -1
  77. package/docs/local-workflow-remediation-2026-07-13.md +0 -130
  78. package/docs/local-workflow-ux-review-2026-07-13.md +0 -458
  79. package/examples/dpo-preferences.jsonl +0 -2
  80. package/examples/dpo-run-request.json +0 -34
  81. package/examples/smoke-run-request.json +0 -34
  82. package/training/local-runner/src/train_dpo.py +0 -286
package/dist/server.js DELETED
@@ -1,211 +0,0 @@
1
- import { createServer } from "node:http";
2
- import { readFile, stat } from "node:fs/promises";
3
- import { createLocalStore } from "./store.js";
4
- import { fineTuneRunRequestSchema, localRunnerConfigSchema } from "./contracts.js";
5
- import { runLocalFineTune } from "./orchestrator.js";
6
- function sendJson(res, status, body) {
7
- res.writeHead(status, {
8
- "Content-Type": "application/json",
9
- "Cache-Control": "no-store",
10
- });
11
- res.end(`${JSON.stringify(body, null, 2)}\n`);
12
- }
13
- function sendText(res, status, body, contentType = "text/plain; charset=utf-8") {
14
- res.writeHead(status, {
15
- "Content-Type": contentType,
16
- "Cache-Control": "no-store",
17
- });
18
- res.end(body);
19
- }
20
- async function readBody(req) {
21
- const chunks = [];
22
- for await (const chunk of req)
23
- chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
24
- const text = Buffer.concat(chunks).toString("utf8").trim();
25
- return text ? JSON.parse(text) : {};
26
- }
27
- const DASHBOARD_HTML = `<!doctype html>
28
- <html lang="en">
29
- <head>
30
- <meta charset="utf-8" />
31
- <meta name="viewport" content="width=device-width, initial-scale=1" />
32
- <title>TT Local</title>
33
- <style>
34
- body { margin: 0; font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #f7f7f4; color: #1e2320; }
35
- header { display: flex; justify-content: space-between; align-items: center; padding: 16px 24px; border-bottom: 1px solid #d9ddd6; background: #ffffff; position: sticky; top: 0; }
36
- main { display: grid; grid-template-columns: minmax(280px, 420px) 1fr; min-height: calc(100vh - 65px); }
37
- aside { border-right: 1px solid #d9ddd6; padding: 16px; overflow: auto; }
38
- section { padding: 20px; overflow: auto; }
39
- button { border: 1px solid #a8b0a8; background: #ffffff; padding: 7px 10px; border-radius: 6px; cursor: pointer; }
40
- .run { width: 100%; text-align: left; display: grid; gap: 4px; margin-bottom: 8px; }
41
- .muted { color: #667066; font-size: 12px; }
42
- .pill { display: inline-block; border: 1px solid #b8c0b6; border-radius: 999px; padding: 2px 8px; font-size: 12px; background: #fff; }
43
- pre { background: #1f2420; color: #eef2ed; padding: 12px; overflow: auto; border-radius: 6px; }
44
- .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 12px; }
45
- .metric { border: 1px solid #d9ddd6; background: #fff; border-radius: 6px; padding: 12px; }
46
- @media (max-width: 760px) { main { grid-template-columns: 1fr; } aside { border-right: 0; border-bottom: 1px solid #d9ddd6; } }
47
- </style>
48
- </head>
49
- <body>
50
- <header>
51
- <strong>TT Local</strong>
52
- <button id="refresh">Refresh</button>
53
- </header>
54
- <main>
55
- <aside><div id="runs"></div></aside>
56
- <section><div id="detail" class="muted">Select a run.</div></section>
57
- </main>
58
- <script>
59
- let selected = null;
60
- let events = null;
61
- const fmt = (v) => v == null ? "—" : String(v);
62
- async function api(path) {
63
- const res = await fetch(path);
64
- if (!res.ok) throw new Error(await res.text());
65
- return res.json();
66
- }
67
- async function loadRuns() {
68
- const runs = await api("/api/runs");
69
- document.getElementById("runs").innerHTML = runs.map(run => \`
70
- <button class="run" data-run="\${run.id}">
71
- <span><strong>\${run.spec_name}</strong> <span class="pill">\${run.status}</span></span>
72
- <span class="muted">\${run.id.slice(0, 8)} · \${run.current_stage} · \${run.base_model}</span>
73
- </button>\`).join("") || '<p class="muted">No runs yet.</p>';
74
- for (const el of document.querySelectorAll("[data-run]")) {
75
- el.onclick = () => showRun(el.getAttribute("data-run"));
76
- }
77
- if (!selected && runs[0]) showRun(runs[0].id);
78
- }
79
- async function showRun(id) {
80
- selected = id;
81
- if (events) events.close();
82
- const run = await api("/api/runs/" + id);
83
- let report = null;
84
- try { report = await api("/api/runs/" + id + "/report"); } catch {}
85
- document.getElementById("detail").innerHTML = \`
86
- <h2>\${run.spec_name}</h2>
87
- <p><span class="pill">\${run.status}</span> <span class="muted">\${run.id}</span></p>
88
- <div class="grid">
89
- <div class="metric"><div class="muted">Stage</div><strong>\${run.current_stage}</strong></div>
90
- <div class="metric"><div class="muted">Base Model</div><strong>\${run.base_model}</strong></div>
91
- <div class="metric"><div class="muted">Model ID</div><strong>\${fmt(run.model_id)}</strong></div>
92
- <div class="metric"><div class="muted">Updated</div><strong>\${run.updated_at}</strong></div>
93
- </div>
94
- <h3>Report</h3>
95
- <pre>\${report ? JSON.stringify({
96
- avg_score_delta: report.comparison?.avg_score_delta,
97
- pass_rate_delta: report.comparison?.pass_rate_delta,
98
- artifact: report.fine_tuned_model_id,
99
- metrics: report.training?.metrics,
100
- }, null, 2) : "Report not available yet."}</pre>
101
- <h3>Events</h3>
102
- <pre id="events"></pre>\`;
103
- const eventBox = document.getElementById("events");
104
- const history = await api("/api/runs/" + id + "/events");
105
- eventBox.textContent = history.map(e => \`\${e.occurred_at} \${e.stage}: \${e.message}\`).join("\\n");
106
- events = new EventSource("/api/runs/" + id + "/events/stream");
107
- events.onmessage = (msg) => {
108
- const e = JSON.parse(msg.data);
109
- eventBox.textContent += "\\n" + \`\${e.occurred_at} \${e.stage}: \${e.message}\`;
110
- };
111
- }
112
- document.getElementById("refresh").onclick = loadRuns;
113
- loadRuns().catch(err => document.getElementById("detail").textContent = err.message);
114
- </script>
115
- </body>
116
- </html>`;
117
- async function streamRunEvents(req, res, storeRoot, runId) {
118
- const store = createLocalStore(storeRoot);
119
- const run = await store.getRun(runId);
120
- const eventPath = `${store.paths.runsDir}/${run.id}/progress.jsonl`;
121
- let offset = 0;
122
- res.writeHead(200, {
123
- "Content-Type": "text/event-stream",
124
- "Cache-Control": "no-cache",
125
- Connection: "keep-alive",
126
- });
127
- const sendNew = async () => {
128
- const size = await stat(eventPath).then((s) => s.size).catch(() => 0);
129
- if (size <= offset)
130
- return;
131
- const text = await readFile(eventPath, "utf8");
132
- const chunk = text.slice(offset);
133
- offset = text.length;
134
- for (const line of chunk.split(/\r?\n/).filter(Boolean)) {
135
- res.write(`data: ${line}\n\n`);
136
- }
137
- };
138
- await sendNew();
139
- const timer = setInterval(() => { sendNew().catch(() => undefined); }, 1000);
140
- req.on("close", () => clearInterval(timer));
141
- }
142
- export async function serveLocalDashboard(options = {}) {
143
- const host = options.host ?? "127.0.0.1";
144
- const port = options.port ?? 8787;
145
- const config = options.config ?? localRunnerConfigSchema.parse({});
146
- const storeRoot = config.storeRoot;
147
- const store = createLocalStore(storeRoot);
148
- await store.ensure();
149
- const server = createServer(async (req, res) => {
150
- try {
151
- const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
152
- const path = url.pathname;
153
- if (req.method === "GET" && path === "/")
154
- return sendText(res, 200, DASHBOARD_HTML, "text/html; charset=utf-8");
155
- if (req.method === "GET" && path === "/api/health")
156
- return sendJson(res, 200, { ok: true, store_root: store.root });
157
- if (req.method === "GET" && path === "/api/runs")
158
- return sendJson(res, 200, await store.listRuns());
159
- if (req.method === "GET" && path === "/api/models")
160
- return sendJson(res, 200, await store.listModels());
161
- if (req.method === "GET" && path === "/api/specs")
162
- return sendJson(res, 200, await store.listSpecs());
163
- const runMatch = path.match(/^\/api\/runs\/([^/]+)(?:\/(events|events\/stream|report|cancel))?$/);
164
- if (runMatch) {
165
- const id = runMatch[1];
166
- const tail = runMatch[2];
167
- if (req.method === "GET" && !tail)
168
- return sendJson(res, 200, await store.getRun(id));
169
- if (req.method === "GET" && tail === "events")
170
- return sendJson(res, 200, await store.getRunEvents(id));
171
- if (req.method === "GET" && tail === "events/stream")
172
- return streamRunEvents(req, res, store.root, id);
173
- if (req.method === "GET" && tail === "report")
174
- return sendJson(res, 200, await store.getRunReport(id));
175
- if (req.method === "POST" && tail === "cancel") {
176
- await store.cancelRun(id);
177
- return sendJson(res, 200, { ok: true });
178
- }
179
- }
180
- const modelMatch = path.match(/^\/api\/models\/([^/]+)$/);
181
- if (req.method === "GET" && modelMatch)
182
- return sendJson(res, 200, await store.getModel(modelMatch[1]));
183
- const specMatch = path.match(/^\/api\/specs\/([^/]+)$/);
184
- if (req.method === "GET" && specMatch)
185
- return sendJson(res, 200, await store.getSpec(specMatch[1]));
186
- if (req.method === "POST" && path === "/api/runs") {
187
- const body = await readBody(req);
188
- const request = fineTuneRunRequestSchema.parse(body.request ?? body);
189
- const runConfig = localRunnerConfigSchema.parse({ ...config, ...(body.config ?? {}) });
190
- void runLocalFineTune({ request, config: runConfig }).catch((error) => {
191
- console.error("[tt-local.serve.run]", error);
192
- });
193
- return sendJson(res, 202, { ok: true, run_id: request.run_id });
194
- }
195
- sendJson(res, 404, { error: "not_found" });
196
- }
197
- catch (error) {
198
- sendJson(res, 500, { error: error instanceof Error ? error.message : String(error) });
199
- }
200
- });
201
- await new Promise((resolveListen, reject) => {
202
- server.once("error", reject);
203
- server.listen(port, host, () => resolveListen());
204
- });
205
- const address = server.address();
206
- return {
207
- url: `http://${host}:${address.port}/`,
208
- close: async () => await new Promise((resolveClose) => server.close(() => resolveClose())),
209
- };
210
- }
211
- //# sourceMappingURL=server.js.map
package/dist/server.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAA6C,MAAM,WAAW,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,wBAAwB,EAAE,uBAAuB,EAA0B,MAAM,gBAAgB,CAAC;AAC3G,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErD,SAAS,QAAQ,CAAC,GAAmB,EAAE,MAAc,EAAE,IAAa;IAClE,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE;QACpB,cAAc,EAAE,kBAAkB;QAClC,eAAe,EAAE,UAAU;KAC5B,CAAC,CAAC;IACH,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,QAAQ,CAAC,GAAmB,EAAE,MAAc,EAAE,IAAY,EAAE,WAAW,GAAG,2BAA2B;IAC5G,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE;QACpB,cAAc,EAAE,WAAW;QAC3B,eAAe,EAAE,UAAU;KAC5B,CAAC,CAAC;IACH,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,GAAoB;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAChG,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3D,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,cAAc,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAyFf,CAAC;AAET,KAAK,UAAU,eAAe,CAAC,GAAoB,EAAE,GAAmB,EAAE,SAAiB,EAAE,KAAa;IACxG,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC,EAAE,iBAAiB,CAAC;IACpE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE;QACjB,cAAc,EAAE,mBAAmB;QACnC,eAAe,EAAE,UAAU;QAC3B,UAAU,EAAE,YAAY;KACzB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;QACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,IAAI,IAAI,IAAI,MAAM;YAAE,OAAO;QAC3B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,EAAE,CAAC;IAChB,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7E,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,UAItC,EAAE;IACJ,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,WAAW,CAAC;IACzC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IACnC,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IAErB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;QAC7C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;YACjF,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;YAC1B,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,cAAc,EAAE,0BAA0B,CAAC,CAAC;YAChH,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,aAAa;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACpH,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,WAAW;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YACpG,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,aAAa;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;YACxG,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,YAAY;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YAEtG,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;YAClG,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACzB,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,IAAI;oBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBACrF,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,QAAQ;oBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvG,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,eAAe;oBAAE,OAAO,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACvG,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,KAAK,QAAQ;oBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvG,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC/C,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBAC1B,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC1D,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,UAAU;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvG,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxD,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,SAAS;gBAAE,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEpG,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,CAA4C,CAAC;gBAC5E,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;gBACrE,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;gBACvF,KAAK,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACpE,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;gBAC/C,CAAC,CAAC,CAAC;gBACH,OAAO,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAClE,CAAC;YAED,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,OAAO,CAAO,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE;QAChD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAiB,CAAC;IAChD,OAAO;QACL,GAAG,EAAE,UAAU,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG;QACtC,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,MAAM,IAAI,OAAO,CAAO,CAAC,YAAY,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;KACjG,CAAC;AACJ,CAAC"}
package/docs/local-workflow-remediation-2026-07-13.md DELETED
@@ -1,130 +0,0 @@
1
- # TT Local workflow remediation
2
-
3
- Date: 2026-07-13
4
- Source review: `docs/local-workflow-ux-review-2026-07-13.md`
5
- Scope: the current, unreleased working-tree patch
6
-
7
- This document records what the patch changes, how the intended first-run flow
8
- should work, and what remains. “Implemented” below means code and focused tests
9
- exist in the working tree. The validation record below distinguishes automated
10
- local evidence from the packaged DGX Spark acceptance run.
11
-
12
- ## Critical findings
13
-
14
- | Finding | Patch status | Remediation in this patch | Remaining qualification |
15
- |---|---|---|---|
16
- | TT-L-001: inconsistent Hugging Face caches | Implemented and warm-cache validated on DGX Spark | `paths.modelCache` now has one meaning: `HF_HOME`. Prefetch, training, evaluation, doctor, and serving resolve the Hub cache beneath `<HF_HOME>/hub`; Python sets cache variables before importing Hugging Face libraries. Prefetch reports the resolved snapshot revision, file count, and bytes. `models verify-base` performs a local-only snapshot check. The packaged Spark run reused the existing Hub snapshot and did not grow the legacy duplicate layout from the prior audit. | The fresh/interrupted/corrupt/gated/unwritable/low-disk matrix is not complete, and prefetch attempts are not indexed for later inspection. |
17
- | TT-L-002: packaged defaults select the wrong Python environment | Implemented and packaged-host validated | Bundled training and Transformers evaluation default to the packaged `training/local-runner` uv project. `doctor` checks the effective commands, required imports, requested CUDA/MPS availability, storage writability/free space, placeholder input, judge configuration, and gated-model token requirements. The package now includes `uv.lock`. A fresh-prefix tarball ran both evaluator and trainer on Linux ARM64/CUDA. | There is no automated Linux ARM64/CUDA CI job, and doctor does not yet estimate whether a selected model and training configuration fit available VRAM. |
18
- | TT-L-003: unsafe resume reuse | Implemented for identified inputs | The run request (including hyperparameters), effective runner/evaluation configuration, dataset and local multimodal-asset contents, TT Local/Node/platform identity, bundled `pyproject.toml`/`uv.lock`, local base snapshot, immutable remote revision, and parent-artifact contents participate in source or per-stage fingerprints. Reuse also verifies manifested outputs; invalidation removes stage-owned stale directories and dependent outputs. A stored `--parent-model` pins or validates the base identity before launch. | A remote base-model branch should still be explicitly pinned by commit for reproducible execution. The broader one-input-at-a-time and interruption matrix remains outstanding. |
19
- | TT-L-004: no model artifact validity contract | Implemented for bundled formats | Real training must produce a non-empty model payload. An atomic `artifact-manifest.json` records the model contract plus file sizes and SHA-256 hashes; gzip/tar integrity is checked for archives. PEFT artifacts require exact adapter weights plus `adapter_config.json`; full-model contracts require recognized Transformers weights. Optimizer-only and incomplete adapters are rejected in fixtures. Dry runs do not create model records. A verified real artifact is registered immediately after training, and `models verify` rechecks the manifest before serving. | A live load/serve smoke test remains the final loadability proof for a particular framework/version combination. Explicit custom command-backend contracts remain caller-defined by design. |
20
- | TT-L-005: unsafe CLI parsing | Implemented | Parsing is strict: top-level and nested help are side-effect free, `--version` is supported, unknown/duplicate/extra flags and missing values fail before work begins, and `--option=value` is accepted. Subprocess-level CLI tests cover the safety contract. | Continue adding each new command and option to the parser contract as the CLI grows. |
21
- | TT-L-006: silent judge fallback | Implemented | `exact_match` is the safe default. Explicit `llm_judge` mode preflights its model/config/key and fails unless `fallback: "exact_match"` was deliberately configured. Fallback results are marked, excluded from judge baseline caching, and judge configuration/key availability participates in cache identity. | A real external-judge acceptance run should still assert `judge_scored_count > 0`; unit tests cannot prove provider availability. |
22
-
23
- ## High-priority findings
24
-
25
- | Gap from the review | Patch response | Status |
26
- |---|---|---|
27
- | A trained artifact is undiscoverable when later evaluation/reporting fails | Register the manifested model immediately after successful training, independently of final report completion. | Implemented and covered by an integration test that forces candidate evaluation failure. |
28
- | `doctor`/`validate` do not preflight the real job | Reject unchanged generated placeholders and invalid judge setups; run exact bundled-environment imports and device checks; check paths, disk, cache, and gated-model credentials. | Implemented; model-fit VRAM estimation remains open. |
29
- | No supported TT Local inference handoff | Add `tt-local models serve`, which verifies the stored adapter and exposes `/health`, `/v1/models`, and `/v1/chat/completions` on localhost by default. Non-loopback binding requires explicit opt-in and bearer-token configuration. | Implemented; the newly trained Spark adapter passed all three live endpoints on CUDA. Streaming remains absent. |
30
- | Cancellation only changes metadata | Poll the cancellation marker at stage boundaries and during long-running evaluation/training; terminate and await the child process group, then publish a terminal `cancelled` state. Late progress/failure/completion writes preserve the request until orchestration finalizes it. | Implemented with process-group, JSON-command, timeout, reporter-failure, and store-race tests; cross-platform process-tree coverage remains to expand. |
31
- | Staged commands leave a nonterminal run | Persist `stage_completed` after an intentionally requested single stage so `runs watch` exits. | Implemented. |
32
- | Successful retry retains stale failure state | Clear superseded run errors when active work restarts or a stage/run succeeds, while retaining event history. | Implemented. |
33
- | Retry/force retains stale model files | Remove only the selected stage's owned directory and dependent outputs before recomputation, then regenerate manifests atomically. | Implemented; interruption-at-each-publication-point tests remain outstanding. |
34
-
35
- ## Improved target workflow
36
-
37
- The following is the intended CUDA/Spark first-run path from an installed
38
- package. Replace the generated prompt/example before validation; TT Local
39
- rejects the unchanged placeholder.
40
-
41
- ```bash
42
- npm install -g @tuned-tensor/local
43
- tt-local --version
44
- uv --version
45
-
46
- tt-local init \
47
- --name "Support Bot" \
48
- --model Qwen/Qwen3.5-2B \
49
- --profile spark
50
-
51
- # Edit tunedtensor.json, then run the exact preflight for this project.
52
- tt-local doctor tunedtensor.json --config local-runner.json
53
- tt-local validate tunedtensor.json --config local-runner.json
54
-
55
- # Make the multi-gigabyte download explicit and verify it without network repair.
56
- tt-local models prefetch tunedtensor.json --config local-runner.json
57
- tt-local models verify-base tunedtensor.json --config local-runner.json
58
-
59
- # Persist the run ID before model work, run in the background, and watch it.
60
- tt-local run tunedtensor.json --config local-runner.json --detach
61
- RUN_ID="<run-id returned by --detach>"
62
- tt-local runs watch "$RUN_ID" --config local-runner.json
63
-
64
- # Inspect results and independently verify the saved model contract.
65
- tt-local runs report "$RUN_ID" --config local-runner.json
66
- MODEL_ID="local-${RUN_ID}"
67
- tt-local models verify "$MODEL_ID" --config local-runner.json
68
-
69
- # Start the model endpoint. `tt-local serve` is the separate run dashboard.
70
- tt-local models serve "$MODEL_ID" \
71
- --config local-runner.json \
72
- --spec tunedtensor.json \
73
- --host 127.0.0.1 \
74
- --port 8000 \
75
- --device cuda
76
- ```
77
-
78
- From a second terminal, check the serving handoff:
79
-
80
- ```bash
81
- curl --fail http://127.0.0.1:8000/health
82
- curl --fail http://127.0.0.1:8000/v1/models
83
- curl --fail http://127.0.0.1:8000/v1/chat/completions \
84
- -H 'Content-Type: application/json' \
85
- -d '{"messages":[{"role":"user","content":"Hello"}],"max_tokens":32}'
86
- ```
87
-
88
- For a gated base model, export `HF_TOKEN` in the shell or an uncommitted project
89
- environment before `doctor`/prefetch. Do not commit credentials, caches, model
90
- artifacts, extracted runtimes, or serving logs.
91
-
92
- ## Validation record
93
-
94
- This record is updated from commands run against this patch. Rows not exercised
95
- are called out explicitly rather than inferred from unit coverage.
96
-
97
- | Validation | Result |
98
- |---|---|
99
- | TypeScript typecheck | Pass: `npm run typecheck`. |
100
- | Node test suite | Pass: `npm test` — 139 passed, 0 failed. |
101
- | Build and package contents | Pass: `npm run build`; `npm pack --dry-run --json` reports 85 entries and includes `training/local-runner/src/serve.py` plus `training/local-runner/uv.lock`. |
102
- | Python syntax and lockfile | Pass: `python3 -m py_compile training/local-runner/src/*.py`; `uv lock --check --project training/local-runner` resolved 85 packages without changing the lock. |
103
- | Fresh-prefix packaged CLI | Pass in `/tmp/tt-local-package-validate.on7l0s`: fresh npm-prefix install, version/help/init/doctor/validate, and full dry run `36cf513f-8846-468f-bca4-4e095f11492e`. |
104
- | DGX Spark cache flow | Pass in `/home/eve/tt-local-patch-test-20260713-BwhiDc`: `HF_HOME=/home/eve/.cache/huggingface`, Hub revision `15852e8c16360a2fea060d615a32b45270f8a8fc`, 10 files, 4,571,198,095 verified bytes. The prior-audit legacy duplicate remained exactly 291,226,208 bytes; the patch created no second layout. |
105
- | DGX Spark real run | Pass on NVIDIA GB10/CUDA: run `0f677a15-5574-4ff1-9aa2-b27780c1c9bd`, model `local-0f677a15-5574-4ff1-9aa2-b27780c1c9bd`, terminal `completed`, 157.992 seconds. `models verify` checked three manifested artifacts and a 35,103,917-byte PEFT tar with base revision pinned. An initial attempt exposed PyTorch 2.13's optional native-JIT dependency on system Python headers; the bundled environment now chooses the eager fallback by default, and the repeat passed. |
106
- | Detach/watch/cancel lifecycle | Pass: successful detached PID 35830 exited after watch completed. Cancellation run `22ee1986-288c-438b-a26e-5f2bc6ebcdcd` moved through `cancel_requested` to terminal `cancelled`; PID 37720 exited and no TT Local/evaluator/trainer process remained. |
107
- | Saved-model serving | Pass on `127.0.0.1:18080`: `/health` reported CUDA, `/v1/models` listed the new model, and chat returned `positive`; Ctrl-C stopped the endpoint and no server process remained. |
108
- | Final diff review | `git diff --check` passes. A second-agent review returned GO with no P0/P1 findings; the modified/untracked secret-pattern scan was clean. |
109
-
110
- ## Honest residual gaps
111
-
112
- The patch does not close the entire acceptance matrix from the review:
113
-
114
- - Prefetch/download attempts are not indexed, so interrupted attempts and cache
115
- history are not discoverable through a dedicated status command.
116
- - CI does not run Linux ARM64 or CUDA; the successful DGX Spark check remains a
117
- manual acceptance environment.
118
- - There is no model-fit VRAM estimate before training.
119
- - Store reports and manifests still need a complete relocation/moved-root
120
- strategy when artifact or store trees are copied elsewhere.
121
- - The OpenAI-compatible serving endpoint does not implement streaming.
122
- - The broader resilience matrix is outstanding: fresh/interrupted/corrupt
123
- downloads, every resume-input mutation, publication-point interruption,
124
- cross-platform process cancellation, moved roots, and missing/corrupt
125
- canonical files.
126
- - Mutable remote model branches cannot be made reproducible by cache inspection
127
- alone; production specs should set an immutable `base_model_revision`.
128
- - Bundled ML subprocesses intentionally receive a minimal environment. Proxy-only
129
- or private-package-index deployments need an explicit, documented credential
130
- forwarding mechanism for prefetch rather than inheriting the whole shell.