@barivia/barsom-mcp 0.10.0 → 0.10.2

package/README.md

@@ -97,11 +97,12 @@ Call at the **start of mapping work** (or when the user asks what the MCP can do
 
 | Action | Use when |
 |--------|----------|
-| `train_map` | Submitting a new map training job — full control: model type, grid, epochs, cyclic/temporal features, transforms. Returns `job_id`; poll with `jobs(action=status, job_id=...)`. |
+| `train_map` | Submitting a new map training job — full control: model type, grid, epochs, cyclic/temporal features, transforms. Returns `job_id`; poll with `jobs(action=status, job_id=...)`. Complete-case data only (no NaNs in training columns). |
+| `train_impute` | Sparse training data: accelerated missSOM / SIOM missSOM — trains a map and imputes missing cells in one job. Defaults: `model=auto`, `cv_folds=5`. Returns map artifacts + `imputed.csv`, `imputation_mask.csv`, optional `quality.csv` / `imputation_uncertainty.csv`. Plain numeric columns only. Valid parent for `inference(impute_column)`. |
 | `train_siom_map` | Submitting a self-interacting map training job — same grid-map workflow plus SIOM controls such as `gamma`, `siom_decay`, and penalty selection. |
 | `train_floop_siom` | Submitting a FLooP-SIOM job — growing node-budget manifold; default `topology=free` (CHL); optional `topology=chain` for strict 1D linked list. |
 | `train_floop_chain` | Deprecated alias for `train_floop_siom` — same behavior. |
-| `status` | Polling after any async job — every 10–15s |
+| `status` | Polling after any async job — every 10–15s; map jobs expose `progress_phase` (ordering, convergence, cv, artifacts) |
 | `list` | Finding job IDs, checking pipeline state |
 | `compare` | Picking the best run from a set (QE, TE, silhouette table) |
 | `cancel` | Stopping a running job |
@@ -127,7 +128,7 @@ All actions use a frozen trained map — no retraining. Derived columns use **`d
 | Action | Output | Timing |
 |--------|--------|--------|
 | `predict` | Score rows against the trained map. **Inputs:** `dataset_id` (defaults to the parent training dataset) **or** inline `rows` (≤500). **Output style** (`output` param): `"compact"` → `predictions.csv` (row_id, bmu_x/y, bmu_node_index, cluster_id [+ QE / qe_p95 / potential_anomaly when scoring **new** data]); `"annotated"` → `annotated.csv` (original CSV + BMU columns appended). **Regime auto-detected:** when the resolved dataset matches the training dataset, QE columns are intentionally omitted in compact output (training-set fit ≠ generalisation; the p95 anomaly flag would be circular). Prefer `dataset_id` for batches and SIOM/irregular maps. | 5–120s |
-| `impute_column` | Fill a numeric **target_column** not used in training: **requires** `dataset_id` + `target_column`. Dataset must contain all training features plus the target. Pools observed target values from rows mapped to this row's BMU and topology neighbors (BMU + neighbors, often 7 nodes on hex interior; fewer on borders unless the map is periodic). `only_missing` (default true); `impute_aggregation`: mean or median. **Not** held-out validated — map-local estimate. Output **`imputed.csv`**. | 5–120s |
+| `impute_column` | Fill a numeric **target_column** not used in training: **requires** `dataset_id` + `target_column`. Parent job: completed **train_map** or **train_impute**. Dataset must contain all training features plus the target. Pools observed target values from rows mapped to this row's BMU and topology neighbors (BMU + neighbors, often 7 nodes on hex interior; fewer on borders unless the map is periodic). `only_missing` (default true); `impute_aggregation`: mean or median. **Not** held-out validated — map-local estimate. Output **`imputed.csv`**. For holes across many training columns, prefer **jobs(train_impute)** first. | 5–120s |
 | `compare` | density-diff heatmap + top gained/lost nodes — drift, A/B, cohort | 30–120s |
 | `project_columns` | Project one or more dataset columns onto the trained map (component planes) | async |
 | `report` | Report **manifest** (figure names, download URLs, metrics, cluster summary) — sync; use with `results(download)` on the training `job_id` for `report.pdf` when present; build custom PDFs in Quarto/Jupyter | immediate |

package/dist/prepare_training_prompt.js

@@ -1,5 +1,5 @@
 import { apiCall } from "./shared.js";
-const FALLBACK_PREFIX = "Please run datasets(action=preview, dataset_id=\"{id}\") to inspect columns, then datasets(action=analyze, dataset_id=\"{id}\") to see which columns and temporal periods are most informative. Then choose the training path: jobs(action=train_map, dataset_id=\"{id}\", ...) for a standard fixed-grid SOM, jobs(action=train_siom_map, dataset_id=\"{id}\", ...) for a fixed-grid SIOM, or jobs(action=train_floop_siom, dataset_id=\"{id}\", ...) for FLooP-SIOM (default topology=free / CHL; optional topology=chain).";
+const FALLBACK_PREFIX = "Before upload: for jobs(action=train_map) ensure training columns have no NaNs; for jobs(action=train_impute) missing cells in training columns are OK (plain numeric only). Please run datasets(action=preview, dataset_id=\"{id}\") to inspect columns, then datasets(action=analyze, dataset_id=\"{id}\") to see which columns and temporal periods are most informative. Then choose the training path: jobs(action=train_map, dataset_id=\"{id}\", ...) for complete-case data, jobs(action=train_impute, dataset_id=\"{id}\", ...) for sparse matrices, jobs(action=train_siom_map, dataset_id=\"{id}\", ...) for a fixed-grid SIOM, or jobs(action=train_floop_siom, dataset_id=\"{id}\", ...) for FLooP-SIOM (default topology=free / CHL; optional topology=chain).";
 /** Used by the `prepare_training` MCP prompt; prefers tier-scoped text from the API when online. */
 export async function resolvePrepareTrainingPromptText(datasetId) {
     let promptText = FALLBACK_PREFIX.replaceAll("{id}", datasetId);

package/dist/shared.js

@@ -19,7 +19,7 @@ export const RETRYABLE_STATUS = new Set([502, 503, 504]);
  * X-Barsom-Client-Version so the server can annotate tool guidance with the
  * wrapper version each action requires. Keep in sync with package.json on bump.
  */
-export const CLIENT_VERSION = "0.10.0";
+export const CLIENT_VERSION = "0.10.2";
 /** User-facing links; keep aligned with barivia.se / api.barivia.se. */
 export const PUBLIC_SITE_ORIGIN = "https://barivia.se";
 /** Poll window for datasets(add_expression) / derive jobs (server-side work can exceed 30s). */

package/dist/tools/datasets.js

@@ -19,8 +19,8 @@ export function registerDatasetsTool(server) {
 
 action=upload: PREFER file_path — server reads from workspace root (token-efficient; no file content in context). Use csv_data only for small inline pastes (e.g. <10KB). Returns dataset ID. Then use datasets(action=preview) before jobs(action=train_map).
 
-Step 0 (before upload): (1) CSV with header; one row per observation. (2) Columns you will use for training must be numeric (or datetime if you will use temporal extraction). (3) No NaNs/missing in those columns (engine does not impute). (4) Categoricals: encode (e.g. one-hot, label) or exclude — do not use raw categorical columns as training features; preview shows which columns are non-numeric. (5) Optional: if you plan to use transition_flow, sort rows chronologically before upload.
-Step 1 (after upload): Upload, then always datasets(action=preview) to verify column types and spot cyclics/datetime. Add derived columns with datasets(action=add_expression, dataset_id=..., name=..., expression=...); fix or subset before jobs(action=train_map).
+Step 0 (before upload): (1) CSV with header; one row per observation. (2) Columns you will use for training must be numeric (or datetime if you will use temporal extraction). (3) Complete-case path (jobs action=train_map): no NaNs in training columns. Sparse path (jobs action=train_impute): missing cells in training columns are OK — the job trains missSOM and returns dense imputed.csv; plain numeric columns only. (4) Categoricals: encode (e.g. one-hot, label) or exclude — do not use raw categorical columns as training features; preview shows which columns are non-numeric. (5) Optional: if you plan to use transition_flow, sort rows chronologically before upload.
+Step 1 (after upload): Upload, then always datasets(action=preview) to verify column types and spot cyclics/datetime. Add derived columns with datasets(action=add_expression, dataset_id=..., name=..., expression=...); choose train_map (complete-case) or train_impute (sparse matrix) before submit.
 
 action=preview: Show columns, stats, sample rows, cyclic/datetime detections. ALWAYS preview before jobs(action=train_map) on an unfamiliar dataset.
 action=analyze: Pre-training analysis on numeric columns — Pearson correlation, autocorrelation periodicity scores, and column recommendations (train, consider_dropping, project_later, low_variance). Use to choose which columns to include in training and which to project onto the map after training.
@@ -42,7 +42,7 @@ action=delete: Remove a dataset and all S3 data permanently.
 
 BEST FOR: Tabular numeric data. CSV with header required.
 NOT FOR: Real-time data streams or binary files — upload a snapshot CSV instead.
-ESCALATION: If upload fails with column errors, open the file locally and verify the header row. If preview shows unexpected nulls, the user must clean the CSV before training.`, {
+ESCALATION: If upload fails with column errors, open the file locally and verify the header row. If preview shows nulls in training columns: use jobs(action=train_impute) for sparse data, or clean/subset for jobs(action=train_map).`, {
         action: z
             .enum(["upload", "preview", "analyze", "list", "subset", "delete", "add_expression", "reduce_spectral"])
             .describe("upload: add CSV; preview: inspect columns/stats; analyze: pre-training correlation and periodicity; list: see all datasets; subset: create filtered subset; delete: remove dataset; add_expression: add derived column from expression; reduce_spectral: collapse a long ordered numeric block (e.g. spectrum, time series) into a small per-row feature set via PCA / log_sample / uniform_sample / stats"),
@@ -256,10 +256,12 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
             const considerDropping = rec.consider_dropping ?? [];
             const projectLater = rec.project_later ?? [];
             const lowVariance = rec.low_variance ?? [];
+            const periodicityCaveat = data.periodicity_caveat;
             const lines = [
                 `Pre-training analysis: ${data.name} (${data.dataset_id})`,
                 `${data.total_rows} rows × ${columns.length} numeric columns analyzed`,
                 ``,
+                ...(periodicityCaveat ? [`Note: ${periodicityCaveat}`, ``] : []),
                 `Column recommendations:`,
                 `  Train (use in jobs(action=train_map)): ${trainList.length ? trainList.join(", ") : "—"}`,
                 `  Consider dropping (highly correlated with another): ${considerDropping.length ? considerDropping.join(", ") : "—"}`,

package/dist/tools/jobs.js

@@ -29,10 +29,10 @@ ESCALATION (action=status):
 - failed → error message and optional failure_stage (e.g. preprocessing, training, metrics, visualization, upload) indicate which phase broke:
   - memory/allocation error: reduce batch_size or grid size and retrain
   - column missing: verify with datasets(action=preview)
-  - NaN error: user must clean the dataset
+  - NaN error on train_map: use jobs(action=train_impute) for sparse training columns, or clean the CSV for complete-case train_map
 
 action=train_map / train_siom_map: Submits a grid-map training job. Returns job_id — poll with jobs(action=status, job_id=...).
-action=train_impute: Submits missing-tolerant map training on a dataset with holes in the feature matrix. Use when many cells are missing across training columns; use inference(impute_column) instead when you already have a complete-case map and need to fill one held-out column. Plain numeric columns only (no cyclic/temporal/categorical). Returns map artifacts plus imputed.csv (dense matrix). Same grid/epochs/backend params as train_map.
+action=train_impute: Submits missing-tolerant map training (accelerated missSOM / SIOM missSOM). Use when many cells are missing across training columns; use inference(impute_column) when you already have a complete-case map and need to fill one held-out column. Plain numeric columns only (no cyclic/temporal/categorical). Defaults: model=auto (SIOM on som_siom+ plans), cv_folds=5 → quality.csv (header feature). status returns progress_phase (ordering/convergence/cv/artifacts). High-dim auto policy: >40 features caps viz; >100 fast metrics; >200 GPU when entitled. Params: viz_mode, viz_top_components, emit_cell_uncertainty, quality_metrics. Artifacts: imputed.csv, imputation_mask.csv, optional imputation_uncertainty.csv. Completed train_impute job_id is a valid parent for inference(impute_column).
   Presets: quick | standard | refined | high_res — use preset=... for grid/epochs/batch defaults; call training_guidance for details.
   Presets refined/high_res may use GPU. On CPU-only hosts pass backend=cpu. API expects strings "cpu" | "gpu" | "gpu_graphs" (no colon). Future backends (e.g. non-CUDA) may be added under the same contract.
   normalize: "auto" (default) = scale only non-cyclic features; "all" = scale every feature. Use "auto" when using cyclic_features.
@@ -269,7 +269,7 @@ export function registerJobsTool(server, description) {
             }
             return v;
         }, z.union([z.number().int(), z.array(z.number().int()).length(2)]).optional()),
-        model: z.enum(["SOM", "RSOM", "SOM-SOFT", "RSOM-SOFT"]).optional().default("SOM"),
+        model: z.enum(["auto", "SOM", "SIOM", "RSOM", "SOM-SOFT", "RSOM-SOFT"]).optional(),
         periodic: z.boolean().optional().default(true),
         columns: z.array(z.string()).optional(),
         cyclic_features: z.array(z.object({
@@ -309,7 +309,13 @@ export function registerJobsTool(server, description) {
         colormap: z.string().optional(),
         row_range: z.tuple([z.number().int().min(1), z.number().int().min(1)]).optional(),
         cv_folds: z.number().int().min(0).max(20).optional()
-            .describe("train_impute only: held-out MAE/RMSE/R2 per column (0=off, 2..20) → quality.csv"),
+            .describe("train_impute only: held-out MAE/RMSE/R2 per column (0=off, default 5) → quality.csv"),
+        viz_mode: z.enum(["full", "summary", "summary_plus_top"]).optional()
+            .describe("Visualization density; auto-capped when feature count > 40"),
+        viz_top_components: z.number().int().min(0).max(64).optional()
+            .describe("With viz_mode=summary_plus_top: upload top-N component maps by variance (default 8)"),
+        emit_cell_uncertainty: z.boolean().optional()
+            .describe("train_impute: write imputation_uncertainty.csv (pool_std per imputed cell)"),
         gamma: z.preprocess((v) => (v !== undefined && v !== null && typeof v === "string") ? parseFloat(v) : v, z.number().optional()),
         gamma_f: z.preprocess((v) => (v !== undefined && v !== null && typeof v === "string") ? parseFloat(v) : v, z.number().optional()),
         siom_decay: z.preprocess((v) => (v !== undefined && v !== null && typeof v === "string") ? parseFloat(v) : v, z.number().optional()),
@@ -366,7 +372,7 @@ export function registerJobsTool(server, description) {
             return { content: [{ type: "text", text: `Baseline study submitted. Job ID: ${jid}\nGrid size: ${side}x${side}\nNormalization: MAD\n\nPoll with jobs(action=status, job_id="${jid}") until complete, then retrieve with results(action=get, job_id="${jid}"). Optional: training_monitor(job_id="${jid}") for a visual panel—not required.` }] };
         }
         if (action === "train_map" || action === "train_siom_map" || action === "train_impute") {
-            const { preset, grid_x, grid_y, epochs, model, periodic, columns, cyclic_features, temporal_features, feature_weights, transforms, auto_log_transforms, time_delay_embeddings, categorical_features, normalize, sigma_f, learning_rate, batch_size, quality_metrics, backend, output_format, output_dpi, colormap, row_range, gamma, gamma_f, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch, siom_feature_geometry, siom_qe_backend, siom_qe_batch_size, label, cv_folds, } = args;
+            const { preset, grid_x, grid_y, epochs, model, periodic, columns, cyclic_features, temporal_features, feature_weights, transforms, auto_log_transforms, time_delay_embeddings, categorical_features, normalize, sigma_f, learning_rate, batch_size, quality_metrics, backend, output_format, output_dpi, colormap, row_range, gamma, gamma_f, siom_decay, siom_penalty, penalty_alpha, reset_per_epoch, siom_feature_geometry, siom_qe_backend, siom_qe_batch_size, label, cv_folds, viz_mode, viz_top_components, emit_cell_uncertainty, } = args;
             let PRESETS = {};
             try {
                 PRESETS = await fetchTrainingPresets();
@@ -389,8 +395,14 @@ export function registerJobsTool(server, description) {
             }, PRESETS);
             if (action === "train_impute") {
                 params._job_type = "train_impute";
-                if (cv_folds !== undefined)
-                    params.cv_folds = cv_folds;
+                params.model = model ?? "auto";
+                params.cv_folds = cv_folds ?? 5;
+                if (viz_mode !== undefined)
+                    params.viz_mode = viz_mode;
+                if (viz_top_components !== undefined)
+                    params.viz_top_components = viz_top_components;
+                if (emit_cell_uncertainty !== undefined)
+                    params.emit_cell_uncertainty = emit_cell_uncertainty;
                 // Plain numeric path only — strip unsupported keys if caller passed them
                 delete params.cyclic_features;
                 delete params.temporal_features;
@@ -567,6 +579,10 @@ export function registerJobsTool(server, description) {
             const label = data.label != null && data.label !== "" ? String(data.label) : null;
             const jobDesc = label ? `Job ${label} (id: ${job_id})` : `Job ${job_id}`;
             let text = `${jobDesc}: ${status} (${progress.toFixed(1)}%)`;
+            const phase = data.progress_phase != null && data.progress_phase !== "" ? String(data.progress_phase) : null;
+            if (phase && status === "running") {
+                text += ` — phase: ${phase}`;
+            }
             if (status === "completed") {
                 text += ` | Results ready. Use results(action=get, job_id="${job_id}") to retrieve.`;
             }

package/dist/tools/results.js

@@ -29,7 +29,7 @@ action=export: Structured data exports. Use export_type= to choose what to expor
   - export_type=nodes: per-node hit count + feature stats. Profile clusters and operating modes.
 
 action=download: Save figures to disk. Use so user can open, share, or version files locally.
-  - folder: e.g. "." or "./results". Interpreted relative to the client's current working directory (or workspace). Files are always written into a per-job subfolder (the job label, else the job_id) under this folder, so downloading several jobs into one folder never overwrites the shared filenames every job emits (e.g. summary.json).
+  - folder: e.g. "." or "./results". Interpreted relative to the client's current working directory (or workspace). Files are written into a per-job subfolder named job_type + label (or job_id) under this folder, so downloading several jobs (even with the same label across types) never overwrites shared filenames like summary.json.
   - figures: "all" (default) or array of filenames.
   - include_json: also save summary.json.
 
@@ -247,6 +247,44 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
                         "Map-local pool estimates — not held-out validated predictions.",
                     ].join("\n") });
             }
+            else if (jobType === "train_impute") {
+                const imp = summary.imputation ?? {};
+                const cv = imp.cv_quality ?? null;
+                const policy = summary.effective_policy ?? {};
+                const hitStats = summary.hit_stats ?? {};
+                const siom = summary.siom ?? undefined;
+                const fmt = (v) => v !== null && v !== undefined ? Number(v).toFixed(4) : "N/A";
+                const lines = [
+                    `missSOM train+impute (${summary.model ?? imp.variant ?? "SOM"}) — ${resultsHeader}`,
+                    `Grid: ${(summary.grid ?? [0, 0]).join("×")} | Features: ${summary.n_features ?? 0} | Samples: ${summary.n_samples ?? 0}`,
+                    `Missing cells: ${imp.n_missing_cells ?? "?"} (${imp.missing_fraction !== undefined ? (Number(imp.missing_fraction) * 100).toFixed(1) + "%" : "?"})`,
+                    `Map quality — QE: ${fmt(summary.quantization_error)} | TE: ${fmt(summary.topographic_error)} | EV: ${fmt(summary.explained_variance)}`,
+                    `Observed-dimension QE: ${fmt(imp.observed_quantization_error)}`,
+                    ...(siom ? [`SIOM — utilization: ${fmt(siom.utilization)} | dead_fraction: ${fmt(siom.dead_fraction)} | siom_qe: ${fmt(siom.siom_qe)}`] : []),
+                    hitStats.grid_suggestion ? `Grid hint: ${String(hitStats.grid_suggestion)}` : "",
+                    ...(policy.auto_rules_applied && Array.isArray(policy.auto_rules_applied) && policy.auto_rules_applied.length > 0
+                        ? [`Auto policy: ${policy.auto_rules_applied.join(", ")} (${policy.viz_mode ?? "viz"}, metrics=${policy.quality_metrics ?? "?"})`]
+                        : []),
+                    cv ? [
+                        "",
+                        `Held-out imputation accuracy (cv_folds=${cv.cv_folds ?? "?"}, method=${cv.cv_method ?? "held_out_prototype_on_trained_map"}):`,
+                        cv.columns_sampled ? `  (sampled ${cv.columns_evaluated ?? "?"} columns for speed)` : "",
+                        cv.aggregate ? `  Aggregate MAE: ${fmt(cv.aggregate.mae)} | RMSE: ${fmt(cv.aggregate.rmse)}` : "",
+                        "  See quality.csv (header column: feature).",
+                    ].filter(Boolean).join("\n") : "\nHeld-out accuracy: skipped (cv_folds=0). Set cv_folds=5 to validate imputation.",
+                    "",
+                    `Artifacts: ${(summary.files ?? []).filter((f) => f !== "summary.json").join(", ")}`,
+                    "Next: results(action=download) for imputed.csv / quality.csv; inference(impute_column) with this job_id as parent for held-out columns.",
+                ].filter((l) => l !== "");
+                content.push({ type: "text", text: lines.join("\n") });
+                for (const name of getResultsImagesToFetch(jobType, summary, figures, include_individual)) {
+                    const cap = getCaptionForImage(name);
+                    if (cap)
+                        content.push({ type: "text", text: cap });
+                    await tryAttachImage(content, job_id, name);
+                    inlinedImages.add(name);
+                }
+            }
             else if (jobType === "reduce_spectral") {
                 const method = String(summary.method ?? "?");
                 const sourceCols = summary.source_columns ?? [];
@@ -581,7 +619,8 @@ NOT FOR: Jobs that haven't completed. Use jobs(action=status) to check first.`,
             // Always namespace each job's files into its own subfolder so that
             // downloading multiple jobs (or job types) into the same folder never
             // overwrites the shared filenames every job emits (e.g. summary.json).
-            const jobSubfolder = (jobLabel ?? job_id).replace(/[^a-zA-Z0-9_.-]/g, "_");
+            // Prefix job_type so the same label on train vs predict/impute/flow jobs cannot collide.
+            const jobSubfolder = `${jobType}_${jobLabel ?? job_id}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
             resolvedDir = path.join(resolvedDir, jobSubfolder);
             if (jobType === "render_variant" && summary.colormap) {
                 const colormapDir = String(summary.colormap).replace(/[^a-zA-Z0-9_-]/g, "_");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@barivia/barsom-mcp",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "barSOM MCP proxy — connect any MCP client to the barSOM cloud API for Self-Organizing Map analytics",
   "keywords": [
     "mcp",