@barivia/barsom-mcp 0.23.4 → 0.23.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/index.js +1 -1
- package/dist/job_monitor.js +31 -7
- package/dist/job_status_format.js +26 -4
- package/dist/shared.js +213 -23
- package/dist/tools/account.js +27 -6
- package/dist/tools/datasets.js +11 -7
- package/dist/tools/explore_map.js +1 -1
- package/dist/tools/inference.js +40 -8
- package/dist/tools/results.js +109 -27
- package/dist/tools/train.js +8 -28
- package/dist/train_submit_message.js +2 -1
- package/dist/training_monitor_curve.js +2 -1
- package/dist/views/src/views/results-explorer/index.html +43 -51
- package/dist/views/src/views/training-monitor/index.html +1 -1
- package/package.json +1 -1
package/dist/tools/account.js
CHANGED
|
@@ -38,7 +38,7 @@ export function registerAccountTool(server) {
|
|
|
38
38
|
| Action | Use when |
|
|
39
39
|
|--------|----------|
|
|
40
40
|
| status | Before large jobs — see plan tier, GPU availability, queue depth, training time estimates, credit balance. When the API is down, returns calm \`{ok:false, retry_after_sec}\` instead of HTML. |
|
|
41
|
-
| history | Viewing recent compute usage and credit spend |
|
|
41
|
+
| history | Viewing recent compute usage and credit spend (gateway-only; local/direct API soft-degrades to "unavailable" — use inventory/jobs list) |
|
|
42
42
|
| inventory | Rebuild a durable markdown overview of datasets + recent jobs (agent-friendly org catalog) |
|
|
43
43
|
| add_funds | Getting instructions to add credits |
|
|
44
44
|
| health | Diagnose API reachability during outages — GET /health + /ready; does **not** probe R2/object storage |
|
|
@@ -157,11 +157,32 @@ NOT FOR: Training itself — use train(action=map). This tool only manages the a
|
|
|
157
157
|
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
158
158
|
}
|
|
159
159
|
if (action === "history") {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
160
|
+
// Gateway serves /v1/compute/history (D1 leases). Local direct API has no handler → 404.
|
|
161
|
+
try {
|
|
162
|
+
const data = await apiCall("GET", `/v1/compute/history?limit=${limit || 10}`);
|
|
163
|
+
const rows = Array.isArray(data?.history) ? data.history : [];
|
|
164
|
+
const history = rows.length === 0
|
|
165
|
+
? "(none)"
|
|
166
|
+
: rows.map((h) => `- ${h.started_at} | ${h.tier} | ${h.duration_minutes} min | $${(h.credits_charged / 100).toFixed(2)}`).join("\n");
|
|
167
|
+
const bal = data?.credit_balance_cents != null
|
|
168
|
+
? `Credit Balance: $${(Number(data.credit_balance_cents) / 100).toFixed(2)}\n\n`
|
|
169
|
+
: "";
|
|
170
|
+
return {
|
|
171
|
+
content: [{ type: "text", text: `${bal}Recent Usage:\n${history}` }]
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
catch (e) {
|
|
175
|
+
const status = e?.httpStatus;
|
|
176
|
+
if (status === 404 || status === 410) {
|
|
177
|
+
return {
|
|
178
|
+
content: [{
|
|
179
|
+
type: "text",
|
|
180
|
+
text: "Usage history unavailable on this API endpoint (compute-lease history is gateway-only). Use account(action=inventory) or jobs(action=list) for recent job activity.",
|
|
181
|
+
}]
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
throw e;
|
|
185
|
+
}
|
|
165
186
|
}
|
|
166
187
|
if (action === "inventory") {
|
|
167
188
|
const jobLimit = limit && limit > 0 ? Math.min(Math.floor(limit), 100) : 50;
|
package/dist/tools/datasets.js
CHANGED
|
@@ -4,7 +4,7 @@ import { gzipSync } from "node:zlib";
|
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { registerAuditedTool } from "../audit.js";
|
|
7
|
-
import { apiCall, getWorkspaceRootAsync, resolveFilePathForUpload, textResult, structuredTextResult, pollUntilComplete, POLL_DERIVE_MAX_MS, POLL_ANALYZE_MAX_MS, UPLOAD_DATASET_TIMEOUT_MS, LARGE_UPLOAD_BYTES, PRESIGNED_PUT_TIMEOUT_MS, POLL_STAGE_MAX_MS, streamFileSha256, putPresignedStream, resolveUploadContentType, suggestDatasetPreview, } from "../shared.js";
|
|
7
|
+
import { apiCall, getWorkspaceRootAsync, resolveFilePathForUpload, textResult, structuredTextResult, pollUntilComplete, POLL_DERIVE_MAX_MS, POLL_ANALYZE_MAX_MS, UPLOAD_DATASET_TIMEOUT_MS, LARGE_UPLOAD_BYTES, PRESIGNED_PUT_TIMEOUT_MS, POLL_STAGE_MAX_MS, awaitDatasetStageIfNeeded, streamFileSha256, putPresignedStream, resolveUploadContentType, suggestDatasetPreview, } from "../shared.js";
|
|
8
8
|
import { GZIP_UPLOAD_HINT } from "../job_status_format.js";
|
|
9
9
|
import { formatDatasetInventoryLine, formatTags } from "../inventory_format.js";
|
|
10
10
|
import { PERIODICITY_ROW_ORDER_NOTE, SMALL_N_SOFT, buildCpuScaleGuidanceLines, buildSmallNGuardrailLines, } from "../training_scale_guidance.js";
|
|
@@ -165,7 +165,7 @@ Step 1 (after upload): always datasets(action=preview) to verify column types an
|
|
|
165
165
|
action=preview: Show columns, stats, sample rows, cyclic/datetime detections. ALWAYS preview before train(action=map) on an unfamiliar dataset.
|
|
166
166
|
action=analyze: Pre-training analysis on numeric columns — Pearson correlation, autocorrelation periodicity scores, and column recommendations (train, consider_dropping, project_later, low_variance). Use to choose which columns to include in training and which to project onto the map after training.
|
|
167
167
|
action=list: List all datasets belonging to the organisation with id, name, rows, cols, created_at, description, tags (use tags/description or id to distinguish datasets with the same name).
|
|
168
|
-
action=get: Fetch one dataset by dataset_id — returns status, staged_prefix, staged_version, ingest_error, description/tags, and stage_job_id when staging is pending.
|
|
168
|
+
action=get: Fetch one dataset by dataset_id — returns status, train_ready (true only when status=ready AND staged_prefix is set), staged_prefix, staged_version, ingest_error, description/tags, and stage_job_id when staging is pending. Train only when train_ready=true (do not treat bare status=ready as enough).
|
|
169
169
|
action=update: PATCH name/description/tags on an existing dataset.
|
|
170
170
|
action=upload: Optional description and tags help later inventory (also settable via update).
|
|
171
171
|
DO NOT AUTO-SUBSET: Call action=subset only when the user explicitly requests a filter, slice, or random sample. Default path is upload → preview/analyze → train on the full dataset_id (GPU handles scale). Never downsample to "save time" or "be helpful".
|
|
@@ -230,7 +230,7 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
|
|
|
230
230
|
})
|
|
231
231
|
.optional()
|
|
232
232
|
.describe("Deprecated — use filters instead. Single filter condition."),
|
|
233
|
-
expression: z.string().optional().describe("action=add_expression: math expression referencing column names (e.g. revenue/cost, log(price), rolling_mean(vol, 20))"),
|
|
233
|
+
expression: z.string().optional().describe("action=add_expression: math expression referencing column names (e.g. revenue/cost, log(price), rolling_mean(vol, 20)). Awkward names (spaces/parens): quote with backticks, e.g. `P1 (bar)` / 2"),
|
|
234
234
|
options: z
|
|
235
235
|
.object({
|
|
236
236
|
missing: z.enum(["skip", "zero", "interpolate"]).optional(),
|
|
@@ -377,7 +377,7 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
|
|
|
377
377
|
gzHeaders["X-Dataset-Description"] = description;
|
|
378
378
|
if (tags !== undefined)
|
|
379
379
|
gzHeaders["X-Dataset-Tags"] = JSON.stringify(tags);
|
|
380
|
-
const data = (await apiCall("POST", "/v1/datasets", gzBytes, gzHeaders, UPLOAD_DATASET_TIMEOUT_MS));
|
|
380
|
+
const data = await awaitDatasetStageIfNeeded((await apiCall("POST", "/v1/datasets", gzBytes, gzHeaders, UPLOAD_DATASET_TIMEOUT_MS)));
|
|
381
381
|
const gid = data.id ?? data.dataset_id;
|
|
382
382
|
if (gid != null)
|
|
383
383
|
data.suggested_next_step = `${suggestDatasetPreview(String(gid))} to inspect columns before training.`;
|
|
@@ -410,7 +410,7 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
|
|
|
410
410
|
uploadBody = gzipSync(Buffer.from(body, "utf-8"));
|
|
411
411
|
uploadHeaders["Content-Encoding"] = "gzip";
|
|
412
412
|
}
|
|
413
|
-
const data = (await apiCall("POST", "/v1/datasets", uploadBody, uploadHeaders, UPLOAD_DATASET_TIMEOUT_MS));
|
|
413
|
+
const data = await awaitDatasetStageIfNeeded((await apiCall("POST", "/v1/datasets", uploadBody, uploadHeaders, UPLOAD_DATASET_TIMEOUT_MS)));
|
|
414
414
|
const id = data.id ?? data.dataset_id;
|
|
415
415
|
if (id != null)
|
|
416
416
|
data.suggested_next_step = `${suggestDatasetPreview(String(id))} to inspect columns before training.`;
|
|
@@ -732,16 +732,20 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
|
|
|
732
732
|
throw new Error("datasets(get) requires dataset_id");
|
|
733
733
|
const ds = (await apiCall("GET", `/v1/datasets/${dataset_id}`));
|
|
734
734
|
const tagStr = formatTags(ds.tags);
|
|
735
|
+
const trainReady = ds.train_ready === true;
|
|
735
736
|
const lines = [
|
|
736
737
|
`Dataset: ${ds.name ?? "?"} (${ds.id ?? dataset_id})`,
|
|
737
|
-
`Status: ${ds.status ?? "
|
|
738
|
+
`Status: ${ds.status ?? "unknown"}`,
|
|
739
|
+
`Train ready: ${trainReady ? "yes" : "no"}`,
|
|
738
740
|
`Rows × cols: ${ds.rows ?? "?"} × ${ds.cols ?? "?"}`,
|
|
739
741
|
ds.size_bytes != null ? `Size: ${Number(ds.size_bytes).toLocaleString()} bytes` : "",
|
|
740
742
|
ds.description != null && String(ds.description).trim() !== "" ? `Description: ${String(ds.description)}` : "",
|
|
741
743
|
tagStr ? `Tags: ${tagStr}` : "",
|
|
742
744
|
ds.staged_prefix != null ? `Staged prefix: ${String(ds.staged_prefix)}` : "",
|
|
743
745
|
ds.staged_version != null ? `Staged version: ${String(ds.staged_version)}` : "",
|
|
744
|
-
ds.stage_job_id != null
|
|
746
|
+
!trainReady && ds.stage_job_id != null
|
|
747
|
+
? `Stage job: ${String(ds.stage_job_id)} (poll jobs(action=status) until train_ready)`
|
|
748
|
+
: "",
|
|
745
749
|
cleanNullable(ds.ingest_error) ? `Ingest error: ${cleanNullable(ds.ingest_error)}` : "",
|
|
746
750
|
ds.created_at != null ? `Created: ${String(ds.created_at)}` : "",
|
|
747
751
|
].filter(Boolean);
|
|
@@ -184,7 +184,7 @@ async function handleResultsExplorer(job_id) {
|
|
|
184
184
|
export function registerExploreMapTool(server) {
|
|
185
185
|
const toolConfig = {
|
|
186
186
|
title: "Results Explorer",
|
|
187
|
-
description: "PREFERRED way to browse a completed map's figures after a first results(get) glance. Interactive explorer
|
|
187
|
+
description: "PREFERRED way to browse a completed map's figures after a first results(get) glance. Interactive explorer with a grouped figure dropdown above the plot (Summary → Diagnostics → Components → Other): combined, U-matrix, hit histogram, component planes, plus sidebar metrics/highlights. Embeds as an MCP App or falls back to a standalone localhost page. Offer it to the user once a train job completes; results(get) remains the headless/metrics path.",
|
|
188
188
|
inputSchema: {
|
|
189
189
|
job_id: z.string().describe("Job ID of a completed map training job"),
|
|
190
190
|
},
|
package/dist/tools/inference.js
CHANGED
|
@@ -79,7 +79,7 @@ action=predict: Score rows against the trained map.
|
|
|
79
79
|
- "compact" → predictions.csv (row_id, bmu_x, bmu_y, bmu_node_index, cluster_id [, quantization_error, potential_anomaly]).
|
|
80
80
|
- "annotated" → annotated.csv (full source CSV with bmu_x, bmu_y, bmu_node_index, cluster_id appended). Requires a dataset (no inline rows).
|
|
81
81
|
Regime (auto): if the resolved dataset is the parent training dataset, regime="training" and QE / qe_p95 / potential_anomaly are omitted (QE on training data is fitting error, not generalisation — use a held-out dataset for quality). Otherwise regime="new" and full QE columns are returned.
|
|
82
|
-
Schema rules:
|
|
82
|
+
Schema rules: prefer dataset_id for scoring. Inline rows may use raw cyclic keys (e.g. hour) — the API/worker expands them from training config to hour_cos/hour_sin; pre-expanded cos/sin columns also work. For a single inline row, the proxy uses the stateless model endpoint (same cyclic expansion). Raw categorical strings are allowed for baseline categorical_features models on that single-row path.
|
|
83
83
|
Routing: prefer dataset_id for many rows or whenever the map uses irregular SIOM / GeneralTopology layouts — the async worker path is the supported batch scorer. Single-row rows take a fast stateless path that may return invalid_inference_input on some topologies; if so, retry with dataset_id (a one-row dataset is fine). FLooP-SIOM: use dataset_id predict first.
|
|
84
84
|
When the scored set has at most ${PREDICT_PREVIEW_ROW_CAP} rows, completed responses include a short per-line preview in the tool text for chat agents.
|
|
85
85
|
|
|
@@ -348,32 +348,64 @@ action=report: Returns a report manifest for the given job_id (job must be compl
|
|
|
348
348
|
const download_urls = (results.download_urls ?? {});
|
|
349
349
|
const figure_manifest = (results.figure_manifest ?? []);
|
|
350
350
|
const cluster_summary = results.cluster_summary;
|
|
351
|
+
const summary = (results.summary ?? {});
|
|
351
352
|
const expires_in = results.expires_in ?? 900;
|
|
353
|
+
const downloadKeys = Object.keys(download_urls);
|
|
354
|
+
const clusterSnippet = Array.isArray(cluster_summary) && cluster_summary.length > 0
|
|
355
|
+
? cluster_summary.slice(0, 3).map((c) => {
|
|
356
|
+
const id = c.cluster_id ?? c.id ?? "?";
|
|
357
|
+
const n = c.n_nodes ?? c.size ?? "?";
|
|
358
|
+
return { cluster_id: id, n_nodes: n };
|
|
359
|
+
})
|
|
360
|
+
: [];
|
|
352
361
|
const lines = [
|
|
353
362
|
`Report manifest — job: ${job_id}`,
|
|
354
363
|
`Use these artifacts to build your own report (Quarto, Jupyter, script). URLs expire in ${expires_in}s.`,
|
|
355
364
|
"",
|
|
356
365
|
"Figures (figure_manifest → download_urls):",
|
|
357
|
-
...figure_manifest.
|
|
366
|
+
...(figure_manifest.length
|
|
367
|
+
? figure_manifest.slice(0, 12).map((e) => ` ${e.logical_name} → ${e.filename}`)
|
|
368
|
+
: [" (none listed — small grids may omit PNGs; use cluster_summary / summary metrics below)"]),
|
|
358
369
|
figure_manifest.length > 12 ? ` ... and ${figure_manifest.length - 12} more` : "",
|
|
359
370
|
"",
|
|
360
371
|
"Key download URLs (use get_result_image or download_urls from results(action=get)):",
|
|
361
|
-
...
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
372
|
+
...(downloadKeys.length
|
|
373
|
+
? downloadKeys
|
|
374
|
+
.filter((k) => !k.endsWith(".json") || k === "cluster_summary.json")
|
|
375
|
+
.slice(0, 8)
|
|
376
|
+
.map((k) => ` ${k}: ${download_urls[k]?.slice(0, 60)}...`)
|
|
377
|
+
: [" (no presigned URLs — artifacts may still be in summary / cluster_summary inline)"]),
|
|
365
378
|
"",
|
|
366
379
|
cluster_summary?.length
|
|
367
|
-
? `Cluster summary: ${cluster_summary.length} clusters (
|
|
380
|
+
? `Cluster summary: ${cluster_summary.length} clusters (inline snippet below; full via GET .../cluster_summary or download_urls.cluster_summary.json).`
|
|
368
381
|
: "Cluster summary: not available for this job (train with current worker to get it).",
|
|
382
|
+
...(clusterSnippet.length
|
|
383
|
+
? ["Inline cluster_summary (first 3):", ...clusterSnippet.map((c) => ` cluster ${c.cluster_id}: n_nodes=${c.n_nodes}`)]
|
|
384
|
+
: []),
|
|
369
385
|
"",
|
|
370
386
|
"Metrics: in summary (quantization_error, topographic_error, explained_variance, silhouette, etc.) or GET .../quality-report.",
|
|
387
|
+
summary.quantization_error != null ? ` QE=${summary.quantization_error}` : "",
|
|
388
|
+
summary.topographic_error != null ? ` TE=${summary.topographic_error}` : "",
|
|
371
389
|
];
|
|
372
390
|
const manifestText = lines.filter(Boolean).join("\n");
|
|
373
391
|
return {
|
|
374
392
|
content: [
|
|
375
393
|
{ type: "text", text: manifestText },
|
|
376
|
-
{
|
|
394
|
+
{
|
|
395
|
+
type: "text",
|
|
396
|
+
text: "Structured manifest (for automation): " +
|
|
397
|
+
JSON.stringify({
|
|
398
|
+
job_id,
|
|
399
|
+
figure_manifest,
|
|
400
|
+
has_cluster_summary: !!cluster_summary?.length,
|
|
401
|
+
cluster_summary_snippet: clusterSnippet,
|
|
402
|
+
download_url_keys: downloadKeys,
|
|
403
|
+
summary_metrics: {
|
|
404
|
+
quantization_error: summary.quantization_error ?? null,
|
|
405
|
+
topographic_error: summary.topographic_error ?? null,
|
|
406
|
+
},
|
|
407
|
+
}),
|
|
408
|
+
},
|
|
377
409
|
],
|
|
378
410
|
};
|
|
379
411
|
});
|
package/dist/tools/results.js
CHANGED
|
@@ -2,8 +2,7 @@ import path from "node:path";
|
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { registerAuditedTool } from "../audit.js";
|
|
5
|
-
import { apiCall, apiRawCall, getWorkspaceRootAsync, sandboxPath, pollUntilComplete, tryAttachImage, resetInlineAttachBudget, attachResultsImages, resolveOutputDpi, POLL_RECOLOR_MAX_MS, fmtDensityDiffNode, getCaptionForImage, shouldFetchAllRemainingFigures, mimeForFilename, structuredTextResult, } from "../shared.js";
|
|
6
|
-
import { formatSiomOccupationSummary } from "../siom_occupation_format.js";
|
|
5
|
+
import { apiCall, apiRawCall, getWorkspaceRootAsync, sandboxPath, pollUntilComplete, tryAttachImage, resetInlineAttachBudget, attachResultsImages, resolveOutputDpi, POLL_RECOLOR_MAX_MS, fmtDensityDiffNode, getCaptionForImage, shouldFetchAllRemainingFigures, mimeForFilename, structuredTextResult, coerceFiguresArg, } from "../shared.js";
|
|
7
6
|
export function registerResultsTool(server) {
|
|
8
7
|
registerAuditedTool(server, "results", `Retrieve, recolor, download, or export figures and metrics for a completed map job. Presentation and artifact access only — operations on the frozen map (predict, compare, project, impute_column, transition_flow) live in **inference**.
|
|
9
8
|
|
|
@@ -13,6 +12,7 @@ export function registerResultsTool(server) {
|
|
|
13
12
|
| export | Learning curve, raw weights, or per-node stats | instant |
|
|
14
13
|
| download | Saving figures to a local folder | instant |
|
|
15
14
|
| recolor | Changing colormap or output format without retraining | async (~10–30s) |
|
|
15
|
+
| feature_divergences | Refresh KL matrices / optional Wasserstein on component planes | async |
|
|
16
16
|
|
|
17
17
|
Temporal state-transition arrows moved to **inference(action=transition_flow)** (use the new result job_id with results(get/download) to read the figure).
|
|
18
18
|
|
|
@@ -20,35 +20,37 @@ ONLY call this after jobs(action=status) returns "completed".
|
|
|
20
20
|
ESCALATION: If job not found, verify job_id. If "job not complete", poll with jobs(action=status).
|
|
21
21
|
|
|
22
22
|
action=get: Returns text summary with quality metrics and inline images.
|
|
23
|
-
- figures: omit = combined only. "all" = all published plots (excludes unpublished cyclic cos/sin expansions unless train used viz_upload_cyclic_components=true). "none" = metrics text only, no images (recommended for sweeps and LLM clients to keep tool payloads small). Array = specific logical names (combined, umatrix, hit_histogram, learning_curve, correlation, component_1..N). Request unpublished cyclic planes via results(recolor, figures=[component_N, ...]).
|
|
23
|
+
- figures: omit = combined only. Prefer string "all" / "none" (not an array). "all" = all published plots (excludes unpublished cyclic cos/sin expansions unless train used viz_upload_cyclic_components=true). "none" = metrics text only, no images (recommended for sweeps and LLM clients to keep tool payloads small). Array = specific logical names (combined, umatrix, hit_histogram, learning_curve, correlation, divergence_kl, component_1..N) or download filenames. Hosts that stringify arrays are coerced. Request unpublished cyclic planes via results(recolor, figures=[component_N, ...]).
|
|
24
24
|
- include_individual: if true (and figures omitted), inlines every component plane.
|
|
25
25
|
- After recolor or project, use the job_id returned by that operation to get the new artifacts, not the original training job_id.
|
|
26
26
|
- After showing results, guide the user: QE interpretation, whether to retrain, which features to explore.
|
|
27
27
|
- METRIC INTERPRETATION: QE lower=better (<1.5 good) | TE<0.1 good | Explained variance>0.7 good | Silhouette higher=better | Davies-Bouldin lower=better.
|
|
28
28
|
|
|
29
|
-
action=export: Structured data exports. Use export_type= to choose what to export.
|
|
30
|
-
- export_type=training_log: learning curve sparklines + plot. Diagnose convergence/plateau/divergence.
|
|
29
|
+
action=export: Structured data exports (not the same as download). Use export_type= to choose what to export.
|
|
30
|
+
- export_type=training_log: learning curve sparklines + plot. Diagnose convergence/plateau/divergence. Prefer this over download for the training log.
|
|
31
31
|
- export_type=weights: full weight matrix + normalization stats. For external analysis or custom viz.
|
|
32
32
|
- export_type=nodes: per-node hit count + feature stats. Profile clusters and operating modes.
|
|
33
33
|
|
|
34
|
-
action=download: Save figures to disk. Use so user can open, share, or version files locally.
|
|
34
|
+
action=download: Save figures/artifacts to disk (deduped file list). Use so user can open, share, or version files locally.
|
|
35
35
|
- folder: e.g. "." or "./results". Relative to the client's working directory (or MCP workspace). Each job lands in a dedicated subfolder under folder: {job_type}_{label} when label was set at train time, otherwise {job_type}_{job_id_prefix} — so parallel downloads never overwrite summary.json or figure names.
|
|
36
|
-
- figures: "all" (default)
|
|
37
|
-
- include_json: when true, also saves summary.json
|
|
36
|
+
- figures: "all" (default) downloads published image files; OR a string array of exact artifact filenames from the job (e.g. ["predictions.csv"], ["combined.png","umatrix.png"]). Enum tokens like "none" are for action=get, not download.
|
|
37
|
+
- include_json: when true, also saves summary.json and other JSON artifacts (use this when you need CSVs listed in summary.files without naming each figure).
|
|
38
38
|
|
|
39
39
|
action=recolor: Change colormap or output format — no retraining. Returns a new job_id; auto-polls 60s.
|
|
40
40
|
AFTER: use results(action=get, job_id=NEW_JOB_ID).
|
|
41
41
|
Colormaps: viridis, plasma, inferno, magma, cividis, turbo, coolwarm, balance, hsv, twilight, RdBu, Spectral. Also: thermal, hot, grays, RdYlBu, delta, curl, phase, rainbow, tol_bright, tol_muted.
|
|
42
42
|
|
|
43
|
+
action=feature_divergences: Recompute component-plane divergences (min-shift sum-normalized denormalized planes). Finalize already writes symmetric KL + rankings; use this for Wasserstein or a refresh. metrics default ["symmetric_kl","directed_kl"]; add "wasserstein" for W1. Auto-polls then summarizes top pairs vs Pearson top_correlations.
|
|
44
|
+
|
|
43
45
|
NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or temporal flow (use inference(action=transition_flow)).`, {
|
|
44
46
|
action: z
|
|
45
|
-
.enum(["get", "recolor", "download", "export"])
|
|
46
|
-
.describe("get: inline results + metrics; recolor: new colormap/format (async); download: save to disk; export: structured data
|
|
47
|
+
.enum(["get", "recolor", "download", "export", "feature_divergences"])
|
|
48
|
+
.describe("get: inline results + metrics; recolor: new colormap/format (async); download: save to disk; export: structured data; feature_divergences: KL/W1 on component planes (async)."),
|
|
47
49
|
job_id: z.string().describe("Job ID of a completed job"),
|
|
48
|
-
figures: z
|
|
50
|
+
figures: z.preprocess(coerceFiguresArg, z
|
|
49
51
|
.union([z.enum(["default", "combined_only", "all", "images", "none"]), z.array(z.string())])
|
|
50
52
|
.optional()
|
|
51
|
-
.describe("action=get: omit=combined only; 'all'=all plots; 'none'=metrics text only
|
|
53
|
+
.describe("Prefer string 'all'/'none'. action=get: omit=combined only; 'all'=all plots; 'none'=metrics text only; array=logical names (combined,umatrix,…). action=download: 'all'=image files; array=exact filenames (e.g. combined.png). Stringified JSON arrays from hosts are accepted.")),
|
|
52
54
|
include_individual: z
|
|
53
55
|
.boolean()
|
|
54
56
|
.optional()
|
|
@@ -80,12 +82,20 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
80
82
|
recolor_figures: z
|
|
81
83
|
.array(z.string())
|
|
82
84
|
.optional()
|
|
83
|
-
.describe("action=recolor: which figures to re-render (default: [combined]). Options: combined, umatrix, hit_histogram, correlation, component_1..N"),
|
|
85
|
+
.describe("action=recolor: which figures to re-render (default: [combined]). Options: combined, umatrix, hit_histogram, correlation, divergence_kl, component_1..N"),
|
|
84
86
|
export_type: z
|
|
85
87
|
.enum(["training_log", "weights", "nodes", "clusters", "transition_matrix"])
|
|
86
88
|
.optional()
|
|
87
89
|
.describe("action=export: training_log=learning curve; weights=weight matrix; nodes=per-node stats; clusters=regime boundaries; transition_matrix=Markov transition matrix"),
|
|
88
|
-
|
|
90
|
+
metrics: z
|
|
91
|
+
.array(z.enum(["symmetric_kl", "directed_kl", "wasserstein"]))
|
|
92
|
+
.optional()
|
|
93
|
+
.describe("action=feature_divergences: which matrices to compute (default symmetric_kl+directed_kl). Add wasserstein for W1 (slower)."),
|
|
94
|
+
wasserstein_method: z
|
|
95
|
+
.enum(["sinkhorn", "exact"])
|
|
96
|
+
.optional()
|
|
97
|
+
.describe("action=feature_divergences: OT solver when metrics includes wasserstein (default sinkhorn)."),
|
|
98
|
+
}, async ({ action, job_id, figures, include_individual, include_json, folder, colormap, output_format, output_dpi, recolor_figures, export_type: exportType, metrics: divergenceMetrics, wasserstein_method: wassersteinMethod }) => {
|
|
89
99
|
const figuresArg = figures;
|
|
90
100
|
if (action === "get") {
|
|
91
101
|
resetInlineAttachBudget();
|
|
@@ -226,10 +236,7 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
226
236
|
hitStats.active_node_fraction !== undefined
|
|
227
237
|
? `Grid BMU occupancy: ${(Number(hitStats.active_node_fraction) * 100).toFixed(1)}% active nodes on the map grid`
|
|
228
238
|
: "",
|
|
229
|
-
...
|
|
230
|
-
files: summary.files ?? [],
|
|
231
|
-
compact: true,
|
|
232
|
-
}),
|
|
239
|
+
...(siom ? [`SIOM layer coverage (distinct from grid dead nodes) — utilization: ${fmt(siom.utilization)} | dead_fraction: ${fmt(siom.dead_fraction)} | siom_qe: ${fmt(siom.siom_qe)}`] : []),
|
|
233
240
|
hitStats.grid_suggestion ? `Grid hint: ${String(hitStats.grid_suggestion)}` : "",
|
|
234
241
|
...(policy.auto_rules_applied && Array.isArray(policy.auto_rules_applied) && policy.auto_rules_applied.length > 0
|
|
235
242
|
? [`Auto policy: ${policy.auto_rules_applied.join(", ")} (${policy.viz_mode ?? "viz"}, metrics=${policy.quality_metrics ?? "?"})`]
|
|
@@ -300,21 +307,37 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
300
307
|
`Coverage Assessment:`,
|
|
301
308
|
` Status: ${coverageStatus}${isDegenerate ? " *** NON-COMPETITIVE ***" : isWarning ? " * CAUTION *" : ""}`,
|
|
302
309
|
` Diagnosis: ${String(coverage.diagnosis ?? "No plain-language diagnosis available.")}`,
|
|
303
|
-
|
|
310
|
+
` Utilization: ${siom.utilization !== undefined ? Number(siom.utilization).toFixed(4) : "N/A"}`,
|
|
311
|
+
` SIOM dead fraction (layer coverage, not grid BMU hits): ${siom.dead_fraction !== undefined ? Number(siom.dead_fraction).toFixed(4) : "N/A"}`,
|
|
304
312
|
` Occupied nodes:${hitStats.occupied_nodes !== undefined ? ` ${hitStats.occupied_nodes}` : " N/A"}`,
|
|
305
313
|
` Top hit share: ${hitStats.top_hit_share !== undefined ? Number(hitStats.top_hit_share).toFixed(4) : "N/A"}`,
|
|
314
|
+
` Gini: ${siom.gini !== undefined ? Number(siom.gini).toFixed(4) : "N/A"}`,
|
|
315
|
+
` Entropy: ${siom.entropy !== undefined ? Number(siom.entropy).toFixed(4) : "N/A"}`,
|
|
316
|
+
` SIOM gamma: ${siom.gamma !== undefined ? Number(siom.gamma).toFixed(3) : "N/A"} | decay: ${siom.decay !== undefined ? Number(siom.decay).toFixed(4) : "N/A"}`,
|
|
306
317
|
...(summary.elastic ? (() => {
|
|
307
318
|
const el = summary.elastic;
|
|
308
319
|
return [` Elastic: λ=${el.lambda !== undefined ? Number(el.lambda).toFixed(3) : "0"} μ=${el.mu !== undefined ? Number(el.mu).toFixed(3) : "0"} anchor=${el.anchor !== undefined ? Number(el.anchor).toFixed(3) : "0"}`];
|
|
309
320
|
})() : []),
|
|
310
321
|
``,
|
|
311
|
-
`Layout tuning: if Voronoi cells look crowded in weight space (high top_hit_share), reduce max_nodes and/or raise gamma/siom_decay before growth_interval. See training_guidance keys floop_voronoi_crowding, floop_siom_params, floop_elastic_params
|
|
322
|
+
`Layout tuning: if Voronoi cells look crowded in weight space (high top_hit_share), reduce max_nodes and/or raise gamma/siom_decay before growth_interval. See training_guidance keys floop_voronoi_crowding, floop_siom_params, floop_elastic_params.`,
|
|
312
323
|
``,
|
|
313
324
|
`Quality Metrics:`,
|
|
314
325
|
` Quantization Error: ${summary.quantization_error !== undefined ? Number(summary.quantization_error).toFixed(4) : "N/A"}${qeNote}`,
|
|
315
326
|
` Topographic Error: ${summary.topographic_error !== undefined ? Number(summary.topographic_error).toFixed(4) : "N/A"}${teNote}`,
|
|
316
327
|
"",
|
|
317
328
|
"Choosing a map for presentation: Lower QE/TE often favors larger, denser graphs. For slides and interpretation, sparser graphs (fewer occupied nodes, lower top_hit_share) are often easier to read even when metrics rank them lower. Compare figures side-by-side before picking a winner from metrics alone.",
|
|
329
|
+
...(() => {
|
|
330
|
+
const topDivs = summary.top_divergences;
|
|
331
|
+
const klComp = summary.feature_kl_complexity;
|
|
332
|
+
const out = [];
|
|
333
|
+
if (topDivs?.length) {
|
|
334
|
+
out.push(`Top component-plane divergences (symmetric KL): ${topDivs.map((t) => `${t.f1}&${t.f2}=${typeof t.symmetric_kl === "number" ? Number(t.symmetric_kl).toFixed(3) : t.symmetric_kl}`).join(", ")}`);
|
|
335
|
+
}
|
|
336
|
+
if (klComp?.length) {
|
|
337
|
+
out.push(`Feature KL complexity: ${klComp.map((t) => `${t.feature}=${typeof t.mean_outgoing_kl === "number" ? Number(t.mean_outgoing_kl).toFixed(3) : t.mean_outgoing_kl}`).join(", ")}`);
|
|
338
|
+
}
|
|
339
|
+
return out.length ? ["", ...out] : [];
|
|
340
|
+
})(),
|
|
318
341
|
...(Array.isArray(coverage.recommended_actions) && coverage.recommended_actions.length > 0
|
|
319
342
|
? ["", "Recommended next steps:", ...coverage.recommended_actions.map((step) => ` - ${String(step)}`)]
|
|
320
343
|
: []),
|
|
@@ -357,7 +380,15 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
357
380
|
: "";
|
|
358
381
|
const topCorrs = summary.top_correlations;
|
|
359
382
|
const topCorrLine = topCorrs && topCorrs.length > 0
|
|
360
|
-
? `Strongest feature correlations: ${topCorrs.map((t) => `${t.f1 ?? "?"} & ${t.f2 ?? "?"} (${typeof t.r === "number" ? Number(t.r).toFixed(2) : t.r})`).join(", ")}`
|
|
383
|
+
? `Strongest feature correlations (Pearson, linear): ${topCorrs.map((t) => `${t.f1 ?? "?"} & ${t.f2 ?? "?"} (${typeof t.r === "number" ? Number(t.r).toFixed(2) : t.r})`).join(", ")}`
|
|
384
|
+
: "";
|
|
385
|
+
const topDivs = summary.top_divergences;
|
|
386
|
+
const topDivLine = topDivs && topDivs.length > 0
|
|
387
|
+
? `Strongest component-plane divergences (symmetric KL, nonlinear): ${topDivs.map((t) => `${t.f1 ?? "?"} & ${t.f2 ?? "?"} (${typeof t.symmetric_kl === "number" ? Number(t.symmetric_kl).toFixed(3) : t.symmetric_kl})`).join(", ")}`
|
|
388
|
+
: "";
|
|
389
|
+
const klComplexity = summary.feature_kl_complexity;
|
|
390
|
+
const klComplexityLine = klComplexity && klComplexity.length > 0
|
|
391
|
+
? `Feature KL complexity (mean outgoing directed KL): ${klComplexity.map((t) => `${t.feature ?? "?"} (${typeof t.mean_outgoing_kl === "number" ? Number(t.mean_outgoing_kl).toFixed(3) : t.mean_outgoing_kl})`).join(", ")}`
|
|
361
392
|
: "";
|
|
362
393
|
const siom = summary.siom;
|
|
363
394
|
const textSummary = [
|
|
@@ -386,11 +417,14 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
386
417
|
...(compLines ? [`Component value ranges:\n${compLines}`] : []),
|
|
387
418
|
...(clusterLines ? ["", "Cluster map (defining features):", clusterLines] : []),
|
|
388
419
|
...(topCorrLine ? ["", topCorrLine] : []),
|
|
389
|
-
...(
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
:
|
|
420
|
+
...(topDivLine ? [topDivLine] : []),
|
|
421
|
+
...(klComplexityLine ? [klComplexityLine] : []),
|
|
422
|
+
...(topDivLine || klComplexityLine ? ["(Compare Pearson vs KL: high |r| ≠ low KL; use results(action=feature_divergences, metrics=[\"wasserstein\"]) for W1.)"] : []),
|
|
423
|
+
...(siom ? ["", "SIOM layer coverage (distinct from grid dead nodes):",
|
|
424
|
+
` Utilization: ${siom.utilization !== undefined ? Number(siom.utilization).toFixed(4) : "N/A"}`,
|
|
425
|
+
` SIOM dead fraction: ${siom.dead_fraction !== undefined ? Number(siom.dead_fraction).toFixed(4) : "N/A"}`,
|
|
426
|
+
` Gini: ${siom.gini !== undefined ? Number(siom.gini).toFixed(4) : "N/A"}`,
|
|
427
|
+
` Entropy: ${siom.entropy !== undefined ? Number(siom.entropy).toFixed(4) : "N/A"}`] : []),
|
|
394
428
|
``, `Features: ${features.join(", ")}`,
|
|
395
429
|
summary.selected_columns ? `Selected columns: ${summary.selected_columns.join(", ")}` : "",
|
|
396
430
|
summary.transforms ? `Transforms: ${Object.entries(summary.transforms).map(([k, v]) => `${k}=${v}`).join(", ")}` : "",
|
|
@@ -434,7 +468,7 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
434
468
|
const showAvailable = files.length > 0 && !(figures === "all" || figures === "images" || figures === "none");
|
|
435
469
|
if (showAvailable) {
|
|
436
470
|
const logicalNames = jobType2 === "train_som" || jobType2 === "train_siom" || jobType2 === "render_variant"
|
|
437
|
-
? `Logical names: combined, umatrix, hit_histogram, correlation, ${featuresForLog.map((_, i) => `component_${i + 1}`).join(", ")}. `
|
|
471
|
+
? `Logical names: combined, umatrix, hit_histogram, correlation, divergence_kl, ${featuresForLog.map((_, i) => `component_${i + 1}`).join(", ")}. `
|
|
438
472
|
: "";
|
|
439
473
|
content.push({ type: "text", text: `Available: ${files.join(", ")}. ${logicalNames}Use results(action=get, figures=[...]) for specific plots, figures=all for everything, or figures="none" for metrics-only.` });
|
|
440
474
|
}
|
|
@@ -597,6 +631,7 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
597
631
|
else {
|
|
598
632
|
toDownload = files.filter(isImage);
|
|
599
633
|
}
|
|
634
|
+
toDownload = [...new Set(toDownload)];
|
|
600
635
|
let resolvedDir = sandboxPath(folder, await getWorkspaceRootAsync(server));
|
|
601
636
|
// Always namespace each job's files into its own subfolder so that
|
|
602
637
|
// downloading multiple jobs (or job types) into the same folder never
|
|
@@ -664,6 +699,53 @@ NOT FOR: Jobs that haven't completed (use jobs(action=status) first), or tempora
|
|
|
664
699
|
}
|
|
665
700
|
return { content: [{ type: "text", text: `Recolor job ${newJobId} submitted. Poll with jobs(action=status, job_id="${newJobId}"), then results(action=get, job_id="${newJobId}").` }] };
|
|
666
701
|
}
|
|
702
|
+
if (action === "feature_divergences") {
|
|
703
|
+
const body = {
|
|
704
|
+
metrics: divergenceMetrics ?? ["symmetric_kl", "directed_kl"],
|
|
705
|
+
output_format: output_format ?? "png",
|
|
706
|
+
output_dpi: resolveOutputDpi(output_dpi),
|
|
707
|
+
};
|
|
708
|
+
if (wassersteinMethod != null)
|
|
709
|
+
body.wasserstein_method = wassersteinMethod;
|
|
710
|
+
const data = (await apiCall("POST", `/v1/results/${job_id}/feature_divergences`, body));
|
|
711
|
+
const newJobId = data.id;
|
|
712
|
+
const poll = await pollUntilComplete(newJobId, POLL_RECOLOR_MAX_MS);
|
|
713
|
+
if (poll.status !== "completed") {
|
|
714
|
+
return {
|
|
715
|
+
content: [{
|
|
716
|
+
type: "text",
|
|
717
|
+
text: `Feature divergences job ${newJobId} submitted (parent ${job_id}). Poll with jobs(action=status, job_id="${newJobId}"), then results(action=get, job_id="${newJobId}").`,
|
|
718
|
+
}],
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
const res = (await apiCall("GET", `/v1/results/${newJobId}`));
|
|
722
|
+
const sum = (res.summary ?? {});
|
|
723
|
+
const topDivs = sum.top_divergences ?? [];
|
|
724
|
+
const klComp = sum.feature_kl_complexity ?? [];
|
|
725
|
+
const topW1 = sum.top_wasserstein ?? null;
|
|
726
|
+
const lines = [
|
|
727
|
+
`Feature divergences complete — job_id: ${newJobId} (parent training: ${job_id})`,
|
|
728
|
+
`Method: ${sum.divergence_method ?? "symmetric_kl_min_shift_sum_norm_denormalized_component_planes"}`,
|
|
729
|
+
`Weight space: ${sum.weight_space ?? "denormalized"}`,
|
|
730
|
+
`Metrics: ${JSON.stringify(sum.metrics ?? body.metrics)}`,
|
|
731
|
+
topDivs.length
|
|
732
|
+
? `Top symmetric KL pairs: ${topDivs.map((t) => `${t.f1}&${t.f2}=${typeof t.symmetric_kl === "number" ? Number(t.symmetric_kl).toFixed(3) : t.symmetric_kl}`).join(", ")}`
|
|
733
|
+
: "",
|
|
734
|
+
klComp.length
|
|
735
|
+
? `KL complexity ranking: ${klComp.map((t) => `${t.feature}=${typeof t.mean_outgoing_kl === "number" ? Number(t.mean_outgoing_kl).toFixed(3) : t.mean_outgoing_kl}`).join(", ")}`
|
|
736
|
+
: "",
|
|
737
|
+
topW1 && topW1.length
|
|
738
|
+
? `Top Wasserstein pairs: ${topW1.map((t) => `${t.f1}&${t.f2}=${typeof t.wasserstein === "number" ? Number(t.wasserstein).toFixed(3) : t.wasserstein}`).join(", ")}`
|
|
739
|
+
: "",
|
|
740
|
+
`Artifacts refreshed on parent result (divergence_kl / feature_divergences.json). Compare with Pearson top_correlations from results(get) on the training job.`,
|
|
741
|
+
].filter(Boolean);
|
|
742
|
+
const content = [{ type: "text", text: lines.join("\n") }];
|
|
743
|
+
const files = sum.files ?? [];
|
|
744
|
+
for (const f of files.filter((x) => /\.(png|svg)$/i.test(x))) {
|
|
745
|
+
await tryAttachImage(content, newJobId, f);
|
|
746
|
+
}
|
|
747
|
+
return { content };
|
|
748
|
+
}
|
|
667
749
|
throw new Error("Invalid action");
|
|
668
750
|
});
|
|
669
751
|
}
|
package/dist/tools/train.js
CHANGED
|
@@ -1,26 +1,25 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { registerAuditedTool } from "../audit.js";
|
|
3
|
-
import { apiCall } from "../shared.js";
|
|
3
|
+
import { apiCall, ensureDatasetTrainReady } from "../shared.js";
|
|
4
4
|
import { buildTrainSubmitExtras } from "../train_submit_message.js";
|
|
5
5
|
import { TRAINING_MONITOR_URI, buildTrainingMonitorResult, } from "./training_monitor.js";
|
|
6
6
|
import { TRAINING_INPUT_SCHEMA, buildTrainMapParams, buildFloopParams, fetchTrainingPresets, assertValidNormalizationArgs, } from "./training_core.js";
|
|
7
|
-
import { SMALL_N_SOFT, buildSmallNGuardrailLines, suggestedAutoGridSide, } from "../training_scale_guidance.js";
|
|
8
7
|
export const TRAIN_DESCRIPTION = `Submit a self-organizing map training job. Returns a job_id; poll with jobs(action=status) and then results(action=get). All actions are async.
|
|
9
8
|
|
|
10
9
|
| Action | Use when |
|
|
11
10
|
|--------|----------|
|
|
12
11
|
| map | Standard SOM on a fixed hex grid — complete-case numeric data (no NaNs in training columns). |
|
|
13
12
|
| siom_map | Self-interacting map — same grid flow plus SIOM coverage control (gamma, siom_decay, penalty) to avoid dead nodes. |
|
|
14
|
-
| impute | Sparse training data: trains a missing-tolerant map (
|
|
13
|
+
| impute | Sparse training data (requires SIOM entitlement): trains a missing-tolerant map (SIOM missSOM when model=auto/SIOM; plain missSOM when model=SOM) AND returns dense imputed.csv in one job. Plain numeric columns only. Optional SIOM knobs: gamma, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch. |
|
|
15
14
|
| floop_siom | FLooP-SIOM — a growing node-budget manifold instead of a fixed grid; default topology=free (CHL). Requires a plan with all_algorithms (Premium/Enterprise). |
|
|
16
15
|
|
|
17
|
-
First quick map (no separate baseline action): use \`train(action=map, preset=quick)\`
|
|
16
|
+
First quick map (no separate baseline action): use \`train(action=map, preset=quick)\` (or \`normalize=mad\` with a grid sized from the preview row count); see the \`baseline_explore\` preset in **training_guidance**.
|
|
18
17
|
|
|
19
18
|
Parameter details (grid, epochs, batch, model, presets, normalize, categorical_features, backend, FLooP max_nodes/effort, SIOM geometry) live in the **training_guidance** tool / **prepare_training** prompt — call them instead of guessing. Backend strings are "cpu" | "gpu" | "gpu_graphs" (no colon); on CPU-only hosts pass backend=cpu.
|
|
20
19
|
|
|
21
|
-
ASYNC: every action returns a job_id immediately. Poll jobs(action=status, job_id=...) every 10–15s (do NOT poll faster). Map training typical times:
|
|
20
|
+
ASYNC: every action returns a job_id immediately. Poll jobs(action=status, job_id=...) every 10–15s (do NOT poll faster). Map training typical times: 10×10 ~30s | 20×20 ~3–5 min | 40×40 ~15–30 min — do not assume failure before 3 minutes on CPU. When status is completed, call results(action=get, job_id=...).
|
|
22
21
|
|
|
23
|
-
action=impute: use when many cells are missing across training columns; use inference(impute_column) instead when you already have a complete-case map and need to fill one held-out column. Supports transforms (log/sqrt/etc., not rank), temporal_features, cyclic_features, optional cyclic_pairs=[[cos,sin],...] (circular prototype refresh + atomic chord BMU; those columns get circular quality metrics only — never quote Euclidean R² on cos/sin), and normalization_methods (zscore/mad/sigmoidal/none — not sepd). Does not support auto_log_transforms, rank transforms, or normalize=sepd. Defaults model=auto (SIOM missSOM
|
|
22
|
+
action=impute: use when many cells are missing across training columns; use inference(impute_column) instead when you already have a complete-case map and need to fill one held-out column. Supports transforms (log/sqrt/etc., not rank), temporal_features, cyclic_features, optional cyclic_pairs=[[cos,sin],...] (circular prototype refresh + atomic chord BMU; those columns get circular quality metrics only — never quote Euclidean R² on cos/sin), and normalization_methods (zscore/mad/sigmoidal/none — not sepd). Does not support auto_log_transforms, rank transforms, or normalize=sepd. Requires SIOM entitlement (train_impute). Defaults model=auto (SIOM missSOM; model=SOM for plain missSOM), cv_folds=5 → quality.csv (linear: kfold_holdout_pool + resubstitution_optimistic; cyclic pairs: kfold_holdout_circular). Aggregate MAE/RMSE excludes cyclic columns. Do not treat k-fold alone as true missing-cell ground truth. Optional SIOM knobs: gamma, gamma_f, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch. Artifacts: imputed.csv, imputation_mask.csv, optional imputation_uncertainty.csv (pool_std = neighbourhood dispersion). A completed impute job_id is a valid parent for inference(impute_column).
|
|
24
23
|
action=floop_siom: max_nodes is a TOTAL node budget (not a grid width/area); if coverage collapses, reduce max_nodes before increasing effort.
|
|
25
24
|
NOT FOR: lifecycle (status/list/compare/cancel/delete) → use jobs; scoring data → use inference.
|
|
26
25
|
|
|
@@ -53,6 +52,7 @@ function assertImputeTrainingArgs(args) {
|
|
|
53
52
|
}
|
|
54
53
|
/** Shared POST /v1/jobs + train submit message extras (map/siom/impute and floop). */
|
|
55
54
|
async function submitTrainJob(opts) {
|
|
55
|
+
await ensureDatasetTrainReady(opts.dataset_id);
|
|
56
56
|
const submitBody = { dataset_id: opts.dataset_id, params: opts.params };
|
|
57
57
|
if (opts.label && opts.label.trim() !== "")
|
|
58
58
|
submitBody.label = opts.label;
|
|
@@ -67,11 +67,6 @@ async function submitTrainJob(opts) {
|
|
|
67
67
|
if (opts.gridLargeForRows) {
|
|
68
68
|
data.grid_auto_warning = true;
|
|
69
69
|
}
|
|
70
|
-
const guardrails = opts.guardrailLines ?? [];
|
|
71
|
-
if (guardrails.length > 0) {
|
|
72
|
-
data.training_guardrails = guardrails;
|
|
73
|
-
data.small_n_warning = opts.totalRows > 0 && opts.totalRows < SMALL_N_SOFT;
|
|
74
|
-
}
|
|
75
70
|
const extras = buildTrainSubmitExtras({
|
|
76
71
|
newJobId,
|
|
77
72
|
totalRows: opts.totalRows,
|
|
@@ -95,12 +90,8 @@ async function submitTrainJob(opts) {
|
|
|
95
90
|
catch {
|
|
96
91
|
/* ignore queue enrichment */
|
|
97
92
|
}
|
|
98
|
-
if (
|
|
99
|
-
msg += ` ${
|
|
100
|
-
}
|
|
101
|
-
else if (opts.gridLargeForRows) {
|
|
102
|
-
const autoSide = suggestedAutoGridSide(opts.totalRows);
|
|
103
|
-
msg += ` Note: Grid may be large for ${opts.totalRows} rows — prefer omitting grid_x/grid_y for auto (~${autoSide}×${autoSide}) to reduce dead nodes.`;
|
|
93
|
+
if (opts.gridLargeForRows) {
|
|
94
|
+
msg += ` Note: Grid may be large for ${opts.totalRows} rows (consider grid=auto for fewer dead nodes).`;
|
|
104
95
|
}
|
|
105
96
|
data.message = msg;
|
|
106
97
|
data.suggested_next_step = extras.suggested_next_step;
|
|
@@ -205,16 +196,6 @@ export async function runTrain(action, args) {
|
|
|
205
196
|
typeof transforms === "object" &&
|
|
206
197
|
Object.keys(transforms).length > 0;
|
|
207
198
|
const gridLargeForRows = effectiveGrid && totalRows > 0 && effectiveGrid[0] * effectiveGrid[1] > totalRows * 0.75;
|
|
208
|
-
const nFeatures = Array.isArray(columns) ? columns.length
|
|
209
|
-
: Array.isArray(params.columns) ? params.columns.length
|
|
210
|
-
: undefined;
|
|
211
|
-
const guardrailLines = buildSmallNGuardrailLines({
|
|
212
|
-
n: totalRows,
|
|
213
|
-
nFeatures,
|
|
214
|
-
preset: typeof preset === "string" ? preset : undefined,
|
|
215
|
-
gridX: effectiveGrid?.[0],
|
|
216
|
-
gridY: effectiveGrid?.[1],
|
|
217
|
-
});
|
|
218
199
|
const resultsHint = action === "impute" ? "view the map and imputed.csv"
|
|
219
200
|
: "view the map and metrics";
|
|
220
201
|
const variantPrefix = action === "siom_map" ? "variant=siom"
|
|
@@ -233,7 +214,6 @@ export async function runTrain(action, args) {
|
|
|
233
214
|
resultsHint,
|
|
234
215
|
impute: action === "impute",
|
|
235
216
|
gridLargeForRows: !!gridLargeForRows,
|
|
236
|
-
guardrailLines,
|
|
237
217
|
});
|
|
238
218
|
}
|
|
239
219
|
if (action === "floop_siom") {
|
|
@@ -2,9 +2,10 @@ import { buildVizStandaloneUrl } from "./viz_links.js";
|
|
|
2
2
|
export function buildTrainSubmitExtras(opts) {
|
|
3
3
|
const { newJobId, totalRows, hasTransforms, prepareJobId, variantPrefix, paramSummary, impute, resultsHint } = opts;
|
|
4
4
|
const monitorUrl = buildVizStandaloneUrl("training-monitor", newJobId);
|
|
5
|
+
const prepareKind = impute ? "prepare_impute_matrix" : "prepare_training_matrix";
|
|
5
6
|
let msg = `Job submitted (${variantPrefix}, ${paramSummary}). `;
|
|
6
7
|
if (prepareJobId) {
|
|
7
|
-
msg += `Preprocessing job
|
|
8
|
+
msg += `Preprocessing job ${prepareKind} (${prepareJobId}) runs on worker-io first; train job ${newJobId} starts after it completes. Poll jobs(action=status, job_id="${prepareJobId}") or jobs(action=status, job_id="${newJobId}") (shows prepare_job_id + waiting_for_prepare) until prepare completes, then poll the train job. `;
|
|
8
9
|
}
|
|
9
10
|
msg += `Training monitor opened for job ${newJobId}`;
|
|
10
11
|
if (monitorUrl)
|
|
@@ -25,7 +25,8 @@ export function teCurveLabel(base) {
|
|
|
25
25
|
* epochs=[N,0], so convergence_errors is empty while ordering has many batch points.
|
|
26
26
|
*/
|
|
27
27
|
export function isFloopJob(data) {
|
|
28
|
-
const
|
|
28
|
+
const rawPhase = String(data.training_progress_phase ?? data.progress_phase ?? "").toLowerCase();
|
|
29
|
+
const phase = rawPhase === "missing" ? "" : rawPhase;
|
|
29
30
|
if (phase === "floop")
|
|
30
31
|
return true;
|
|
31
32
|
const jt = String(data.job_type ?? "").toLowerCase();
|