@barivia/barsom-mcp 0.15.4 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -7
- package/dist/index.js +1 -1
- package/dist/job_status_format.js +52 -13
- package/dist/shared.js +25 -4
- package/dist/tools/datasets.js +24 -1
- package/dist/tools/explore_map.js +1 -1
- package/dist/tools/inference.js +43 -4
- package/dist/tools/jobs.js +16 -56
- package/dist/tools/results.js +34 -62
- package/dist/tools/train.js +73 -133
- package/dist/tools/training_core.js +126 -9
- package/dist/tools/training_guidance.js +1 -1
- package/dist/tools/training_monitor.js +49 -4
- package/dist/tools/training_prep.js +211 -21
- package/dist/views/src/views/training-monitor/index.html +29 -16
- package/dist/views/src/views/training-prep/index.html +25 -11
- package/dist/viz-server.js +19 -0
- package/package.json +1 -1
package/dist/tools/results.js
CHANGED
|
@@ -4,7 +4,7 @@ import { z } from "zod";
|
|
|
4
4
|
import { registerAuditedTool } from "../audit.js";
|
|
5
5
|
import { apiCall, apiRawCall, getWorkspaceRootAsync, sandboxPath, pollUntilComplete, tryAttachImage, getResultsImagesToFetch, getCaptionForImage, mimeForFilename, } from "../shared.js";
|
|
6
6
|
export function registerResultsTool(server) {
|
|
7
|
-
registerAuditedTool(server, "results", `Retrieve, recolor, download, export
|
|
7
|
+
registerAuditedTool(server, "results", `Retrieve, recolor, download, or export figures and metrics for a completed map job. Presentation and artifact access only — operations on the frozen map (predict, compare, project, impute_column, transition_flow) live in **inference**.
|
|
8
8
|
|
|
9
9
|
| Action | Use when | Sync/Async |
|
|
10
10
|
|--------|----------|------------|
|
|
@@ -12,7 +12,8 @@ export function registerResultsTool(server) {
|
|
|
12
12
|
| export | Learning curve, raw weights, or per-node stats | instant |
|
|
13
13
|
| download | Saving figures to a local folder | instant |
|
|
14
14
|
| recolor | Changing colormap or output format without retraining | async (~10–30s) |
|
|
15
|
-
|
|
15
|
+
|
|
16
|
+
Temporal state-transition arrows moved to **inference(action=transition_flow)** (use the new result job_id with results(get/download) to read the figure).
|
|
16
17
|
|
|
17
18
|
ONLY call this after jobs(action=status) returns "completed".
|
|
18
19
|
ESCALATION: If job not found, verify job_id. If "job not complete", poll with jobs(action=status).
|
|
@@ -38,15 +39,10 @@ action=recolor: Change colormap or output format — no retraining. Returns a ne
|
|
|
38
39
|
AFTER: use results(action=get, job_id=NEW_JOB_ID).
|
|
39
40
|
Colormaps: viridis, plasma, inferno, magma, cividis, turbo, coolwarm, balance, hsv, twilight, RdBu, Spectral. Also: thermal, hot, grays, RdYlBu, delta, curl, phase, rainbow, tol_bright, tol_muted.
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
- You pass the training job_id; the API returns a new result job_id. Use that new job_id for results(get) or results(download).
|
|
43
|
-
- To persist the figure: results(action=download, job_id=<returned_job_id>).
|
|
44
|
-
- lag=1 (default): immediate next-step | lag=N: N-step horizon (e.g. 24 for daily cycles in hourly data).
|
|
45
|
-
- min_transitions: filter noisy arrows. Increase for large datasets.
|
|
46
|
-
NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`, {
|
|
42
|
+
NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or temporal flow (use inference(action=transition_flow)).`, {
|
|
47
43
|
action: z
|
|
48
|
-
.enum(["get", "recolor", "download", "export"
|
|
49
|
-
.describe("get: inline results + metrics; recolor: new colormap/format (async); download: save to disk; export: structured data (training_log/weights/nodes)
|
|
44
|
+
.enum(["get", "recolor", "download", "export"])
|
|
45
|
+
.describe("get: inline results + metrics; recolor: new colormap/format (async); download: save to disk; export: structured data (training_log/weights/nodes). Temporal flow moved to inference(action=transition_flow)."),
|
|
50
46
|
job_id: z.string().describe("Job ID of a completed job"),
|
|
51
47
|
figures: z
|
|
52
48
|
.union([z.enum(["default", "combined_only", "all", "images", "none"]), z.array(z.string())])
|
|
@@ -69,12 +65,12 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
69
65
|
colormap: z
|
|
70
66
|
.string()
|
|
71
67
|
.optional()
|
|
72
|
-
.describe("action=recolor: colormap name (default: coolwarm).
|
|
68
|
+
.describe("action=recolor: colormap name (default: coolwarm). Examples: viridis, plasma, balance, hsv, twilight, RdBu, phase, rainbow."),
|
|
73
69
|
output_format: z
|
|
74
70
|
.enum(["png", "pdf", "svg"])
|
|
75
71
|
.optional()
|
|
76
72
|
.default("png")
|
|
77
|
-
.describe("action=recolor
|
|
73
|
+
.describe("action=recolor: output image format (default: png)"),
|
|
78
74
|
output_dpi: z
|
|
79
75
|
.enum(["standard", "retina", "print"])
|
|
80
76
|
.optional()
|
|
@@ -88,19 +84,7 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
88
84
|
.enum(["training_log", "weights", "nodes", "clusters", "transition_matrix"])
|
|
89
85
|
.optional()
|
|
90
86
|
.describe("action=export: training_log=learning curve; weights=weight matrix; nodes=per-node stats; clusters=regime boundaries; transition_matrix=Markov transition matrix"),
|
|
91
|
-
|
|
92
|
-
.number().int().min(1)
|
|
93
|
-
.optional().default(1)
|
|
94
|
-
.describe("action=transition_flow: step lag (default 1 = consecutive rows). Use larger for periodic analysis (e.g. 24 for daily in hourly data)."),
|
|
95
|
-
min_transitions: z
|
|
96
|
-
.number().int().min(1)
|
|
97
|
-
.optional()
|
|
98
|
-
.describe("action=transition_flow: minimum transition count to draw an arrow (default: auto). Increase to filter noise."),
|
|
99
|
-
top_k: z
|
|
100
|
-
.number().int().min(1)
|
|
101
|
-
.optional().default(10)
|
|
102
|
-
.describe("action=transition_flow: number of top-flow nodes in statistics (default 10)"),
|
|
103
|
-
}, async ({ action, job_id, figures, include_individual, include_json, folder, colormap, output_format, output_dpi, recolor_figures, export_type: exportType, lag, min_transitions, top_k }) => {
|
|
87
|
+
}, async ({ action, job_id, figures, include_individual, include_json, folder, colormap, output_format, output_dpi, recolor_figures, export_type: exportType }) => {
|
|
104
88
|
const dpiMap = { standard: 1, retina: 2, print: 4 };
|
|
105
89
|
if (action === "get") {
|
|
106
90
|
const data = (await apiCall("GET", `/v1/results/${job_id}`));
|
|
@@ -261,7 +245,10 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
261
245
|
`Missing cells: ${imp.n_missing_cells ?? "?"} (${imp.missing_fraction !== undefined ? (Number(imp.missing_fraction) * 100).toFixed(1) + "%" : "?"})`,
|
|
262
246
|
`Map quality — QE: ${fmt(summary.quantization_error)} | TE: ${fmt(summary.topographic_error)} | EV: ${fmt(summary.explained_variance)}`,
|
|
263
247
|
`Observed-dimension QE: ${fmt(imp.observed_quantization_error)}`,
|
|
264
|
-
|
|
248
|
+
hitStats.active_node_fraction !== undefined
|
|
249
|
+
? `Grid BMU occupancy: ${(Number(hitStats.active_node_fraction) * 100).toFixed(1)}% active nodes on the map grid`
|
|
250
|
+
: "",
|
|
251
|
+
...(siom ? [`SIOM layer coverage (distinct from grid dead nodes) — utilization: ${fmt(siom.utilization)} | dead_fraction: ${fmt(siom.dead_fraction)} | siom_qe: ${fmt(siom.siom_qe)}`] : []),
|
|
265
252
|
hitStats.grid_suggestion ? `Grid hint: ${String(hitStats.grid_suggestion)}` : "",
|
|
266
253
|
...(policy.auto_rules_applied && Array.isArray(policy.auto_rules_applied) && policy.auto_rules_applied.length > 0
|
|
267
254
|
? [`Auto policy: ${policy.auto_rules_applied.join(", ")} (${policy.viz_mode ?? "viz"}, metrics=${policy.quality_metrics ?? "?"})`]
|
|
@@ -334,15 +321,24 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
334
321
|
` Status: ${coverageStatus}${isDegenerate ? " *** NON-COMPETITIVE ***" : isWarning ? " * CAUTION *" : ""}`,
|
|
335
322
|
` Diagnosis: ${String(coverage.diagnosis ?? "No plain-language diagnosis available.")}`,
|
|
336
323
|
` Utilization: ${siom.utilization !== undefined ? Number(siom.utilization).toFixed(4) : "N/A"}`,
|
|
337
|
-
`
|
|
324
|
+
` SIOM dead fraction (layer coverage, not grid BMU hits): ${siom.dead_fraction !== undefined ? Number(siom.dead_fraction).toFixed(4) : "N/A"}`,
|
|
338
325
|
` Occupied nodes:${hitStats.occupied_nodes !== undefined ? ` ${hitStats.occupied_nodes}` : " N/A"}`,
|
|
339
326
|
` Top hit share: ${hitStats.top_hit_share !== undefined ? Number(hitStats.top_hit_share).toFixed(4) : "N/A"}`,
|
|
340
327
|
` Gini: ${siom.gini !== undefined ? Number(siom.gini).toFixed(4) : "N/A"}`,
|
|
341
328
|
` Entropy: ${siom.entropy !== undefined ? Number(siom.entropy).toFixed(4) : "N/A"}`,
|
|
329
|
+
` SIOM gamma: ${siom.gamma !== undefined ? Number(siom.gamma).toFixed(3) : "N/A"} | decay: ${siom.decay !== undefined ? Number(siom.decay).toFixed(4) : "N/A"}`,
|
|
330
|
+
...(summary.elastic ? (() => {
|
|
331
|
+
const el = summary.elastic;
|
|
332
|
+
return [` Elastic: λ=${el.lambda !== undefined ? Number(el.lambda).toFixed(3) : "0"} μ=${el.mu !== undefined ? Number(el.mu).toFixed(3) : "0"} anchor=${el.anchor !== undefined ? Number(el.anchor).toFixed(3) : "0"}`];
|
|
333
|
+
})() : []),
|
|
334
|
+
``,
|
|
335
|
+
`Layout tuning: if Voronoi cells look crowded in weight space (high top_hit_share), reduce max_nodes and/or raise gamma/siom_decay before growth_interval. See training_guidance keys floop_voronoi_crowding, floop_siom_params, floop_elastic_params.`,
|
|
342
336
|
``,
|
|
343
337
|
`Quality Metrics:`,
|
|
344
338
|
` Quantization Error: ${summary.quantization_error !== undefined ? Number(summary.quantization_error).toFixed(4) : "N/A"}${qeNote}`,
|
|
345
339
|
` Topographic Error: ${summary.topographic_error !== undefined ? Number(summary.topographic_error).toFixed(4) : "N/A"}${teNote}`,
|
|
340
|
+
"",
|
|
341
|
+
"Choosing a map for presentation: Lower QE/TE often favors larger, denser graphs. For slides and interpretation, sparser graphs (fewer occupied nodes, lower top_hit_share) are often easier to read even when metrics rank them lower. Compare figures side-by-side before picking a winner from metrics alone.",
|
|
346
342
|
...(Array.isArray(coverage.recommended_actions) && coverage.recommended_actions.length > 0
|
|
347
343
|
? ["", "Recommended next steps:", ...coverage.recommended_actions.map((step) => ` - ${String(step)}`)]
|
|
348
344
|
: []),
|
|
@@ -371,7 +367,7 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
371
367
|
const nNodes = grid[0] * grid[1];
|
|
372
368
|
const hitStats = summary.hit_stats;
|
|
373
369
|
const hitLine = hitStats
|
|
374
|
-
? `
|
|
370
|
+
? ` Grid BMU occupancy: ${hitStats.dead_nodes ?? "N/A"} dead of ${nNodes} (${nNodes ? ((Number(hitStats.dead_nodes) / nNodes) * 100).toFixed(1) : "0"}% dead${hitStats.active_node_fraction !== undefined ? `, ${(Number(hitStats.active_node_fraction) * 100).toFixed(1)}% active` : ""}) | Max hits: ${hitStats.max_hits ?? "N/A"} | Median: ${hitStats.median_hits ?? "N/A"}`
|
|
375
371
|
: "";
|
|
376
372
|
const uRange = summary.u_matrix_range;
|
|
377
373
|
const uRangeLine = uRange && uRange.length >= 2 ? ` U-matrix range: ${fmt(uRange[0])} — ${fmt(uRange[1])}` : "";
|
|
@@ -399,6 +395,12 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
399
395
|
`Grid: ${grid[0]}×${grid[1]} | Features: ${summary.n_features ?? 0} | Samples: ${summary.n_samples ?? 0}`,
|
|
400
396
|
`Model: ${summary.model ?? "SOM"} | Epochs: ${epochStr}`,
|
|
401
397
|
`Periodic: ${summary.periodic ?? true} | Normalize: ${summary.normalize ?? "auto"}`,
|
|
398
|
+
(() => {
|
|
399
|
+
const audit = summary.normalization_audit;
|
|
400
|
+
if (!audit?.length)
|
|
401
|
+
return "";
|
|
402
|
+
return `Normalization audit: ${audit.map((a) => `${String(a.feature)}: requested ${String(a.requested)}, applied ${String(a.applied)} (${String(a.reason)})`).join("; ")}`;
|
|
403
|
+
})(),
|
|
402
404
|
summary.sigma_f !== undefined ? `Sigma_f: ${summary.sigma_f}` : "",
|
|
403
405
|
duration !== undefined ? `Training duration: ${duration}s` : "",
|
|
404
406
|
``, `Quality Metrics:`,
|
|
@@ -409,14 +411,14 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
409
411
|
` Davies-Bouldin: ${fmt(summary.davies_bouldin)} (lower is better)`,
|
|
410
412
|
` Calinski-Harabasz: ${fmt(summary.calinski_harabasz)} (higher is better)`,
|
|
411
413
|
ordErrors && ordErrors.length > 0 ? ` Final ordering QE: ${ordErrors.at(-1)?.toFixed(4)} (use results(action=export, export_type=training_log) for full curve)` : "",
|
|
412
|
-
...(hitLine ? ["", "Hit / U-matrix:", hitLine] : []),
|
|
414
|
+
...(hitLine ? ["", "Hit / U-matrix (grid BMU occupancy):", hitLine] : []),
|
|
413
415
|
...(uRangeLine ? [uRangeLine] : []),
|
|
414
416
|
...(compLines ? [`Component value ranges:\n${compLines}`] : []),
|
|
415
417
|
...(clusterLines ? ["", "Cluster map (defining features):", clusterLines] : []),
|
|
416
418
|
...(topCorrLine ? ["", topCorrLine] : []),
|
|
417
|
-
...(siom ? ["", "SIOM coverage
|
|
419
|
+
...(siom ? ["", "SIOM layer coverage (distinct from grid dead nodes):",
|
|
418
420
|
` Utilization: ${siom.utilization !== undefined ? Number(siom.utilization).toFixed(4) : "N/A"}`,
|
|
419
|
-
`
|
|
421
|
+
` SIOM dead fraction: ${siom.dead_fraction !== undefined ? Number(siom.dead_fraction).toFixed(4) : "N/A"}`,
|
|
420
422
|
` Gini: ${siom.gini !== undefined ? Number(siom.gini).toFixed(4) : "N/A"}`,
|
|
421
423
|
` Entropy: ${siom.entropy !== undefined ? Number(siom.entropy).toFixed(4) : "N/A"}`] : []),
|
|
422
424
|
``, `Features: ${features.join(", ")}`,
|
|
@@ -432,7 +434,7 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
432
434
|
return "";
|
|
433
435
|
return `Columns not used in training: ${excluded.join(", ")}. You can project them onto this map with inference(action=project_columns, job_id=${job_id}, dataset_id=<dataset_id>, columns=[${excluded.map((c) => `"${c}"`).join(", ")}]) to see how they distribute across the topology. If you ran datasets(action=analyze) before training, any columns it recommended as "project later" are especially good candidates.`;
|
|
434
436
|
})(),
|
|
435
|
-
...((jobType === "train_som" || jobType === "train_siom" || jobType === "train_impute") ? ["", `Next: results(action=export, export_type=training_log) for learning curve; results(action=download) to save figures${jobType === "train_impute" ? " (includes imputed.csv)" : ""}; jobs(action=compare, job_ids=[...]) to compare runs; inference(action=predict) to score new data; inference(action=project_columns) to project other variables onto the map.`] : []),
|
|
437
|
+
...((jobType === "train_som" || jobType === "train_siom" || jobType === "train_impute") ? ["", `Next: results_explorer(job_id="${job_id}") to browse all figures interactively; results(action=export, export_type=training_log) for learning curve; results(action=download) to save figures${jobType === "train_impute" ? " (includes imputed.csv)" : ""}; jobs(action=compare, job_ids=[...]) to compare runs; inference(action=predict) to score new data; inference(action=project_columns) to project other variables onto the map.`] : []),
|
|
436
438
|
].filter((l) => l !== "").join("\n");
|
|
437
439
|
content.push({ type: "text", text: textSummary });
|
|
438
440
|
const imagesToFetch = getResultsImagesToFetch(jobType, summary, figures, include_individual);
|
|
@@ -663,36 +665,6 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
|
|
|
663
665
|
}
|
|
664
666
|
return { content: [{ type: "text", text: `Recolor job ${newJobId} submitted. Poll with jobs(action=status, job_id="${newJobId}"), then results(action=get, job_id="${newJobId}").` }] };
|
|
665
667
|
}
|
|
666
|
-
if (action === "transition_flow") {
|
|
667
|
-
const body = { lag: lag ?? 1, output_format: output_format ?? "png" };
|
|
668
|
-
if (min_transitions !== undefined)
|
|
669
|
-
body.min_transitions = min_transitions;
|
|
670
|
-
if (top_k !== undefined)
|
|
671
|
-
body.top_k = top_k;
|
|
672
|
-
if (colormap !== undefined)
|
|
673
|
-
body.colormap = colormap;
|
|
674
|
-
if (output_dpi && output_dpi !== "retina")
|
|
675
|
-
body.output_dpi = dpiMap[output_dpi] ?? 2;
|
|
676
|
-
const data = (await apiCall("POST", `/v1/results/${job_id}/transition-flow`, body));
|
|
677
|
-
const flowJobId = data.id;
|
|
678
|
-
const poll = await pollUntilComplete(flowJobId, 120_000);
|
|
679
|
-
if (poll.status === "completed") {
|
|
680
|
-
const res = (await apiCall("GET", `/v1/results/${flowJobId}`));
|
|
681
|
-
const summary = (res.summary ?? {});
|
|
682
|
-
const stats = (summary.flow_stats ?? {});
|
|
683
|
-
const content = [{ type: "text", text: [
|
|
684
|
-
`Transition Flow (job: ${flowJobId}) | Parent map: ${job_id} | Lag: ${lag ?? 1}`,
|
|
685
|
-
`Active flow nodes: ${stats.active_flow_nodes ?? "N/A"} | Total transitions: ${stats.total_transitions ?? "N/A"}`,
|
|
686
|
-
`Mean magnitude: ${stats.mean_magnitude !== undefined ? Number(stats.mean_magnitude).toFixed(4) : "N/A"}`,
|
|
687
|
-
].join("\n") }];
|
|
688
|
-
await tryAttachImage(content, flowJobId, `transition_flow_lag${lag ?? 1}.${output_format ?? "png"}`);
|
|
689
|
-
return { content };
|
|
690
|
-
}
|
|
691
|
-
if (poll.status === "failed") {
|
|
692
|
-
return { content: [{ type: "text", text: `Transition flow job ${flowJobId} failed: ${poll.error ?? "unknown error"}` }] };
|
|
693
|
-
}
|
|
694
|
-
return { content: [{ type: "text", text: `Transition flow job ${flowJobId} submitted. Poll with jobs(action=status, job_id="${flowJobId}"), retrieve with results(action=get, job_id="${flowJobId}").` }] };
|
|
695
|
-
}
|
|
696
668
|
throw new Error("Invalid action");
|
|
697
669
|
});
|
|
698
670
|
}
|
package/dist/tools/train.js
CHANGED
|
@@ -2,53 +2,58 @@ import { z } from "zod";
|
|
|
2
2
|
import { registerAuditedTool } from "../audit.js";
|
|
3
3
|
import { apiCall, textResult } from "../shared.js";
|
|
4
4
|
import { buildTrainSubmitExtras } from "../train_submit_message.js";
|
|
5
|
-
import { TRAINING_INPUT_SCHEMA, buildTrainMapParams, fetchTrainingPresets, } from "./training_core.js";
|
|
5
|
+
import { TRAINING_INPUT_SCHEMA, buildTrainMapParams, buildFloopParams, fetchTrainingPresets, } from "./training_core.js";
|
|
6
6
|
export const TRAIN_DESCRIPTION = `Submit a self-organizing map training job. Returns a job_id; poll with jobs(action=status) and then results(action=get). All actions are async.
|
|
7
7
|
|
|
8
8
|
| Action | Use when |
|
|
9
9
|
|--------|----------|
|
|
10
10
|
| map | Standard SOM on a fixed hex grid — complete-case numeric data (no NaNs in training columns). |
|
|
11
11
|
| siom_map | Self-interacting map — same grid flow plus SIOM coverage control (gamma, siom_decay, penalty) to avoid dead nodes. |
|
|
12
|
-
| impute | Sparse training data: trains a missing-tolerant map (accelerated missSOM) AND returns dense imputed.csv in one job. Plain numeric columns only. |
|
|
12
|
+
| impute | Sparse training data: trains a missing-tolerant map (accelerated missSOM; SIOM missSOM when model=auto on som_siom+ or model=SIOM) AND returns dense imputed.csv in one job. Plain numeric columns only. Optional SIOM knobs: gamma, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch. |
|
|
13
13
|
| floop_siom | FLooP-SIOM — a growing node-budget manifold instead of a fixed grid; default topology=free (CHL). Requires a plan with all_algorithms (Premium/Enterprise). |
|
|
14
|
-
|
|
14
|
+
|
|
15
|
+
First quick map (no separate baseline action): use \`train(action=map, preset=quick)\` (or \`normalize=mad\` with a grid sized from the preview row count); see the \`baseline_explore\` preset in **training_guidance**.
|
|
15
16
|
|
|
16
17
|
Parameter details (grid, epochs, batch, model, presets, normalize, categorical_features, backend, FLooP max_nodes/effort, SIOM geometry) live in the **training_guidance** tool / **prepare_training** prompt — call them instead of guessing. Backend strings are "cpu" | "gpu" | "gpu_graphs" (no colon); on CPU-only hosts pass backend=cpu.
|
|
17
18
|
|
|
18
19
|
ASYNC: every action returns a job_id immediately. Poll jobs(action=status, job_id=...) every 10–15s (do NOT poll faster). Map training typical times: 10×10 ~30s | 20×20 ~3–5 min | 40×40 ~15–30 min — do not assume failure before 3 minutes on CPU. When status is completed, call results(action=get, job_id=...).
|
|
19
20
|
|
|
20
|
-
action=impute: use when many cells are missing across training columns; use inference(impute_column) instead when you already have a complete-case map and need to fill one held-out column. Defaults model=auto, cv_folds=5 → quality.csv with two labeled numbers per column (kfold_holdout_pool = honest held-out; resubstitution_optimistic = in-sample, optimistic). Artifacts: imputed.csv, imputation_mask.csv, optional imputation_uncertainty.csv. A completed impute job_id is a valid parent for inference(impute_column).
|
|
21
|
+
action=impute: use when many cells are missing across training columns; use inference(impute_column) instead when you already have a complete-case map and need to fill one held-out column. Supports transforms (log/sqrt/etc., not rank), temporal_features, cyclic_features, and normalization_methods (zscore/mad/sigmoidal/none — not sepd). Does not support auto_log_transforms, rank transforms, or normalize=sepd. Defaults model=auto (SIOM missSOM on som_siom+ plans, else plain missSOM), cv_folds=5 → quality.csv with two labeled numbers per column (kfold_holdout_pool = honest held-out; resubstitution_optimistic = in-sample, optimistic). Optional SIOM coverage knobs: gamma, gamma_f, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch (same names as siom_map). Artifacts: imputed.csv, imputation_mask.csv, optional imputation_uncertainty.csv, summary.siom when SIOM path runs. A completed impute job_id is a valid parent for inference(impute_column).
|
|
21
22
|
action=floop_siom: max_nodes is a TOTAL node budget (not a grid width/area); if coverage collapses, reduce max_nodes before increasing effort.
|
|
22
|
-
NOT FOR: lifecycle (status/list/compare/cancel/delete) → use jobs; scoring data → use inference
|
|
23
|
+
NOT FOR: lifecycle (status/list/compare/cancel/delete) → use jobs; scoring data → use inference.
|
|
24
|
+
|
|
25
|
+
Rank transform (map/siom/floop only): training-only — batch predict, project_columns, and impute_column cannot score rank-transformed columns.`;
|
|
26
|
+
function assertImputeTrainingArgs(args) {
|
|
27
|
+
const normalize = args.normalize;
|
|
28
|
+
if (normalize === "sepd") {
|
|
29
|
+
throw new Error("train(impute) does not support normalize=sepd");
|
|
30
|
+
}
|
|
31
|
+
const nm = args.normalization_methods;
|
|
32
|
+
if (nm) {
|
|
33
|
+
for (const [col, method] of Object.entries(nm)) {
|
|
34
|
+
if (method === "sepd") {
|
|
35
|
+
throw new Error(`train(impute) does not support normalization_methods.${col}=sepd`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const transforms = args.transforms;
|
|
40
|
+
if (transforms) {
|
|
41
|
+
for (const [col, t] of Object.entries(transforms)) {
|
|
42
|
+
if (t === "rank") {
|
|
43
|
+
throw new Error("train(impute) rank transform is not supported");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (args.auto_log_transforms === true) {
|
|
48
|
+
throw new Error("train(impute) does not support auto_log_transforms yet.");
|
|
49
|
+
}
|
|
50
|
+
}
|
|
23
51
|
/**
|
|
24
52
|
* Core training dispatcher. Shared by the `train` tool and the deprecated
|
|
25
53
|
* `jobs(train_*)` wrappers. Every branch posts to the same /v1/jobs route.
|
|
26
54
|
*/
|
|
27
55
|
export async function runTrain(action, args) {
|
|
28
56
|
const dataset_id = args.dataset_id;
|
|
29
|
-
if (action === "baseline_study") {
|
|
30
|
-
if (!dataset_id)
|
|
31
|
-
throw new Error("train(baseline_study) requires dataset_id");
|
|
32
|
-
const dsInfo = (await apiCall("GET", `/v1/datasets/${dataset_id}/preview?n_rows=1`));
|
|
33
|
-
const nSamples = dsInfo.total_rows || 1000;
|
|
34
|
-
const totalNodes = Math.round(5 * Math.sqrt(nSamples));
|
|
35
|
-
const side = Math.max(5, Math.round(Math.sqrt(totalNodes)));
|
|
36
|
-
const params = {
|
|
37
|
-
model: "SOM",
|
|
38
|
-
periodic: false,
|
|
39
|
-
normalize: "mad",
|
|
40
|
-
grid: [side, side],
|
|
41
|
-
epochs: [10, 20],
|
|
42
|
-
batch_size: 32,
|
|
43
|
-
quality_metrics: "standard",
|
|
44
|
-
output_format: "png",
|
|
45
|
-
output_dpi: 2,
|
|
46
|
-
colormap: "coolwarm",
|
|
47
|
-
};
|
|
48
|
-
const data = (await apiCall("POST", "/v1/jobs", { dataset_id, params }));
|
|
49
|
-
const jid = String(data.id ?? "");
|
|
50
|
-
return { content: [{ type: "text", text: `Baseline study submitted. Job ID: ${jid}\nGrid size: ${side}x${side}\nNormalization: MAD\n\nPoll with jobs(action=status, job_id="${jid}") until complete, then retrieve with results(action=get, job_id="${jid}"). Optional: training_monitor(job_id="${jid}") for a visual panel—not required.` }] };
|
|
51
|
-
}
|
|
52
57
|
if (action === "map" || action === "siom_map" || action === "impute") {
|
|
53
58
|
const { preset, grid_x, grid_y, epochs, model, periodic, columns, cyclic_features, temporal_features, feature_weights, transforms, auto_log_transforms, time_delay_embeddings, categorical_features, normalize, sigma_f, learning_rate, batch_size, quality_metrics, backend, output_format, output_dpi, colormap, row_range, gamma, gamma_f, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch, siom_feature_geometry, siom_qe_backend, siom_qe_batch_size, label, cv_folds, viz_mode, viz_top_components, emit_cell_uncertainty, } = args;
|
|
54
59
|
let PRESETS = {};
|
|
@@ -64,6 +69,9 @@ export async function runTrain(action, args) {
|
|
|
64
69
|
}
|
|
65
70
|
if (!dataset_id)
|
|
66
71
|
throw new Error(`train(${action}) requires dataset_id`);
|
|
72
|
+
if (action === "impute") {
|
|
73
|
+
assertImputeTrainingArgs(args);
|
|
74
|
+
}
|
|
67
75
|
const { params, paramSummary, effectiveGrid } = buildTrainMapParams({
|
|
68
76
|
preset, grid_x, grid_y, epochs, model, periodic, columns, cyclic_features,
|
|
69
77
|
temporal_features, feature_weights, transforms, auto_log_transforms,
|
|
@@ -81,34 +89,44 @@ export async function runTrain(action, args) {
|
|
|
81
89
|
params.viz_top_components = viz_top_components;
|
|
82
90
|
if (emit_cell_uncertainty !== undefined)
|
|
83
91
|
params.emit_cell_uncertainty = emit_cell_uncertainty;
|
|
84
|
-
//
|
|
85
|
-
delete params.cyclic_features;
|
|
86
|
-
delete params.temporal_features;
|
|
92
|
+
// Strip keys still rejected on impute (transforms/temporal/cyclic/normalization_methods are kept)
|
|
87
93
|
delete params.categorical_features;
|
|
88
|
-
delete params.transforms;
|
|
89
94
|
delete params.auto_log_transforms;
|
|
90
95
|
delete params.time_delay_embeddings;
|
|
91
96
|
}
|
|
92
97
|
if (action === "siom_map") {
|
|
93
98
|
params._job_type = "train_siom";
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
99
|
+
}
|
|
100
|
+
if (action === "siom_map" || action === "impute") {
|
|
101
|
+
const imputeModel = action === "impute" ? String(params.model ?? "auto") : "";
|
|
102
|
+
const siomKnobSet = gamma !== undefined ||
|
|
103
|
+
gamma_f !== undefined ||
|
|
104
|
+
siom_decay !== undefined ||
|
|
105
|
+
siom_penalty !== undefined ||
|
|
106
|
+
penalty_alpha !== undefined ||
|
|
107
|
+
reset_per_epoch !== undefined;
|
|
108
|
+
if (action === "siom_map" || imputeModel.toUpperCase() === "SIOM" || siomKnobSet) {
|
|
109
|
+
if (gamma !== undefined)
|
|
110
|
+
params.gamma = gamma;
|
|
111
|
+
if (gamma_f !== undefined)
|
|
112
|
+
params.gamma_f = gamma_f;
|
|
113
|
+
if (siom_decay !== undefined)
|
|
114
|
+
params.siom_decay = siom_decay;
|
|
115
|
+
if (siom_penalty !== undefined)
|
|
116
|
+
params.siom_penalty = siom_penalty;
|
|
117
|
+
if (penalty_alpha !== undefined)
|
|
118
|
+
params.penalty_alpha = penalty_alpha;
|
|
119
|
+
if (reset_per_epoch !== undefined)
|
|
120
|
+
params.reset_per_epoch = reset_per_epoch;
|
|
121
|
+
}
|
|
122
|
+
if (action === "siom_map") {
|
|
123
|
+
if (siom_feature_geometry !== undefined)
|
|
124
|
+
params.siom_feature_geometry = siom_feature_geometry;
|
|
125
|
+
if (siom_qe_backend !== undefined)
|
|
126
|
+
params.siom_qe_backend = siom_qe_backend;
|
|
127
|
+
if (siom_qe_batch_size !== undefined)
|
|
128
|
+
params.siom_qe_batch_size = siom_qe_batch_size;
|
|
129
|
+
}
|
|
112
130
|
}
|
|
113
131
|
let totalRows = 0;
|
|
114
132
|
try {
|
|
@@ -125,8 +143,7 @@ export async function runTrain(action, args) {
|
|
|
125
143
|
: action === "impute" ? "variant=train_impute"
|
|
126
144
|
: "variant=som";
|
|
127
145
|
data.effective_params = `${variantPrefix}, ${paramSummary}`;
|
|
128
|
-
const hasTransforms =
|
|
129
|
-
transforms != null &&
|
|
146
|
+
const hasTransforms = transforms != null &&
|
|
130
147
|
typeof transforms === "object" &&
|
|
131
148
|
Object.keys(transforms).length > 0;
|
|
132
149
|
const extras = buildTrainSubmitExtras({
|
|
@@ -164,79 +181,10 @@ export async function runTrain(action, args) {
|
|
|
164
181
|
return textResult(data);
|
|
165
182
|
}
|
|
166
183
|
if (action === "floop_siom") {
|
|
167
|
-
const { columns, cyclic_features, temporal_features, feature_weights, transforms, auto_log_transforms, normalize, row_range, output_format, output_dpi, colormap, topology, max_nodes, effort, n_initial, n_passes, growth_interval, neighborhood_size, edge_max_age, max_degree, gamma, siom_decay, siom_penalty, penalty_alpha, error_threshold, error_decay, ring_close_threshold, sigma_0, sigma_f, eta_0, eta_f, elastic_lambda, elastic_mu, elastic_anchor, anchor_percentile, label, } = args;
|
|
168
184
|
if (!dataset_id)
|
|
169
185
|
throw new Error("train(floop_siom) requires dataset_id");
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
topology,
|
|
173
|
-
effort,
|
|
174
|
-
normalize,
|
|
175
|
-
output_format: output_format ?? "png",
|
|
176
|
-
};
|
|
177
|
-
if (max_nodes !== undefined)
|
|
178
|
-
params.max_nodes = max_nodes;
|
|
179
|
-
if (columns?.length)
|
|
180
|
-
params.columns = columns;
|
|
181
|
-
if (cyclic_features?.length)
|
|
182
|
-
params.cyclic_features = cyclic_features;
|
|
183
|
-
if (temporal_features?.length)
|
|
184
|
-
params.temporal_features = temporal_features;
|
|
185
|
-
if (feature_weights && Object.keys(feature_weights).length > 0)
|
|
186
|
-
params.feature_weights = feature_weights;
|
|
187
|
-
if (transforms && Object.keys(transforms).length > 0)
|
|
188
|
-
params.transforms = transforms;
|
|
189
|
-
if (auto_log_transforms)
|
|
190
|
-
params.auto_log_transforms = auto_log_transforms;
|
|
191
|
-
if (row_range && row_range.length >= 2 && row_range[0] <= row_range[1])
|
|
192
|
-
params.row_range = row_range;
|
|
193
|
-
const dpiMap = { standard: 1, retina: 2, print: 4 };
|
|
194
|
-
if (output_dpi && output_dpi !== "retina")
|
|
195
|
-
params.output_dpi = dpiMap[output_dpi] ?? 2;
|
|
196
|
-
if (colormap)
|
|
197
|
-
params.colormap = colormap;
|
|
198
|
-
if (n_initial !== undefined)
|
|
199
|
-
params.n_initial = n_initial;
|
|
200
|
-
if (n_passes !== undefined)
|
|
201
|
-
params.n_passes = n_passes;
|
|
202
|
-
if (growth_interval !== undefined)
|
|
203
|
-
params.growth_interval = growth_interval;
|
|
204
|
-
if (neighborhood_size !== undefined)
|
|
205
|
-
params.neighborhood_size = neighborhood_size;
|
|
206
|
-
if (edge_max_age !== undefined)
|
|
207
|
-
params.edge_max_age = edge_max_age;
|
|
208
|
-
if (max_degree !== undefined)
|
|
209
|
-
params.max_degree = max_degree;
|
|
210
|
-
if (gamma !== undefined)
|
|
211
|
-
params.gamma = gamma;
|
|
212
|
-
if (siom_decay !== undefined)
|
|
213
|
-
params.siom_decay = siom_decay;
|
|
214
|
-
if (siom_penalty !== undefined)
|
|
215
|
-
params.siom_penalty = siom_penalty;
|
|
216
|
-
if (penalty_alpha !== undefined)
|
|
217
|
-
params.penalty_alpha = penalty_alpha;
|
|
218
|
-
if (error_threshold !== undefined)
|
|
219
|
-
params.error_threshold = error_threshold;
|
|
220
|
-
if (error_decay !== undefined)
|
|
221
|
-
params.error_decay = error_decay;
|
|
222
|
-
if (ring_close_threshold !== undefined)
|
|
223
|
-
params.ring_close_threshold = ring_close_threshold;
|
|
224
|
-
if (sigma_0 !== undefined)
|
|
225
|
-
params.sigma_0 = sigma_0;
|
|
226
|
-
if (sigma_f !== undefined)
|
|
227
|
-
params.sigma_f = sigma_f;
|
|
228
|
-
if (eta_0 !== undefined)
|
|
229
|
-
params.eta_0 = eta_0;
|
|
230
|
-
if (eta_f !== undefined)
|
|
231
|
-
params.eta_f = eta_f;
|
|
232
|
-
if (elastic_lambda !== undefined)
|
|
233
|
-
params.elastic_lambda = elastic_lambda;
|
|
234
|
-
if (elastic_mu !== undefined)
|
|
235
|
-
params.elastic_mu = elastic_mu;
|
|
236
|
-
if (elastic_anchor !== undefined)
|
|
237
|
-
params.elastic_anchor = elastic_anchor;
|
|
238
|
-
if (anchor_percentile !== undefined)
|
|
239
|
-
params.anchor_percentile = anchor_percentile;
|
|
186
|
+
const { transforms, label } = args;
|
|
187
|
+
const { params, paramSummary } = buildFloopParams(args);
|
|
240
188
|
const submitBody = { dataset_id, params };
|
|
241
189
|
if (label && label.trim() !== "")
|
|
242
190
|
submitBody.label = label;
|
|
@@ -249,14 +197,6 @@ export async function runTrain(action, args) {
|
|
|
249
197
|
}
|
|
250
198
|
catch { /* ignore */ }
|
|
251
199
|
const hasTransforms = transforms != null && typeof transforms === "object" && Object.keys(transforms).length > 0;
|
|
252
|
-
const maxNodeSummary = max_nodes === undefined ? "max_nodes=auto(~2*sqrt(n_samples))" : `max_nodes=${max_nodes}`;
|
|
253
|
-
const paramSummary = [
|
|
254
|
-
"variant=floop",
|
|
255
|
-
`topology=${topology ?? "free"}`,
|
|
256
|
-
maxNodeSummary,
|
|
257
|
-
`effort=${effort ?? "standard"}`,
|
|
258
|
-
gamma !== undefined ? `gamma=${gamma}` : "",
|
|
259
|
-
].filter(Boolean).join(", ");
|
|
260
200
|
data.effective_params = paramSummary;
|
|
261
201
|
const extras = buildTrainSubmitExtras({
|
|
262
202
|
newJobId,
|
|
@@ -276,8 +216,8 @@ export async function runTrain(action, args) {
|
|
|
276
216
|
export function registerTrainTool(server) {
|
|
277
217
|
registerAuditedTool(server, "train", TRAIN_DESCRIPTION, {
|
|
278
218
|
action: z
|
|
279
|
-
.enum(["map", "siom_map", "impute", "floop_siom"
|
|
280
|
-
.describe("map: standard SOM; siom_map: self-interacting map; impute: missing-tolerant map + imputed.csv; floop_siom: growing manifold (Premium/Enterprise)
|
|
219
|
+
.enum(["map", "siom_map", "impute", "floop_siom"])
|
|
220
|
+
.describe("map: standard SOM; siom_map: self-interacting map; impute: missing-tolerant map + imputed.csv; floop_siom: growing manifold (Premium/Enterprise). For a quick first map use action=map with preset=quick."),
|
|
281
221
|
dataset_id: z.string().optional().describe("Dataset ID to train on (required for all actions)."),
|
|
282
222
|
label: z
|
|
283
223
|
.string()
|