@barivia/barsom-mcp 0.10.5 → 0.11.1

package/README.md

@@ -42,7 +42,7 @@ MCP clients typically run it with **`npx`** (downloads on first use):
 | `BARIVIA_API_KEY` | Yes | -- | API key (starts with `bv_`) |
 | `BARIVIA_API_URL` | No | `https://api.barivia.se` | API base URL |
 | `BARIVIA_WORKSPACE_ROOT` | No | `process.cwd()` or `PWD` | Directory for relative `file_path` and `save_to_disk`. In Cursor MCP, `process.cwd()` is often the MCP install dir — add `BARIVIA_WORKSPACE_ROOT` to your MCP config `env` with your project path (e.g. `/home/user/myproject`). Absolute paths and `file://` URIs work without it. |
-| `BARIVIA_FETCH_TIMEOUT_MS` | No | `30000` | Per-request HTTP timeout (ms) to the API. Increase (e.g. `120000`) on slow networks or when the API does long-running work. Large dataset uploads use a separate longer timeout internally. |
+| `BARIVIA_FETCH_TIMEOUT_MS` | No | `60000` | Per-request HTTP timeout (ms) to the API. Increase (e.g. `120000`) on slow networks or when the API does long-running work. Large dataset uploads use a separate longer timeout internally, and `datasets(analyze)` runs asynchronously (auto-polled) so it is not bound by this. A timeout is not auto-retried (re-firing slow work would only multiply load / risk duplicates). |
 | `BARIVIA_VIZ_PORT` | No | OS-assigned | When the client has no MCP Apps support, the proxy may start a local viz server on `127.0.0.1`; set a fixed port if you need stable bookmark URLs. |
 | `BARIVIA_ENFORCE_WORKSPACE_SANDBOX` | No | `1` (enabled) | File uploads are constrained to the MCP workspace root by default. Set to `0` or `false` to allow absolute paths anywhere on the machine (high trust). **Changed in v0.x:** previously defaulted to off; now on for security. If uploads fail with "paths outside the workspace are disabled", either set `BARIVIA_WORKSPACE_ROOT` to cover your data directory, or opt out with `BARIVIA_ENFORCE_WORKSPACE_SANDBOX=0`. |
 
@@ -203,7 +203,7 @@ MCP Client (Cursor/Claude) ←stdio→ @barivia/barsom-mcp ←HTTPS→ api.bariv
 | Symptom | What to check |
 |--------|----------------|
 | `401` / invalid key | `BARIVIA_API_KEY` in MCP config; check your email or manage your key in the [account dashboard](https://barivia.se/dashboard). Error text includes a **request id** for support. |
-| Request timed out | Raise `BARIVIA_FETCH_TIMEOUT_MS` (e.g. `120000`). Large uploads already use an extended timeout. |
+| Request timed out | Raise `BARIVIA_FETCH_TIMEOUT_MS` (e.g. `120000`). Large uploads already use an extended timeout; `datasets(analyze)` is async (auto-polled). Timeouts are not auto-retried, so re-running a timed-out upload is safe — an idempotency key reconciles it to the original dataset instead of duplicating. |
 | `Path must be within the workspace` / upload can’t find file | Set `BARIVIA_WORKSPACE_ROOT` to your project directory, or use an absolute path / `file:///...` URI. |
 | Job stuck “running” | Poll `jobs(action=status)` every 10–15s; large grids or FLooP-SIOM can take several minutes—not an MCP error. |
 | `429` | Rate limit—wait and retry. |

package/dist/index.js

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-import{McpServer as e}from"@modelcontextprotocol/sdk/server/mcp.js";import{StdioServerTransport as t}from"@modelcontextprotocol/sdk/server/stdio.js";import{z as o}from"zod";import{getUiCapability as r,registerAppResource as n,RESOURCE_MIME_TYPE as s}from"@modelcontextprotocol/ext-apps/server";import{startVizServer as a}from"./viz-server.js";import{API_KEY as i,apiCall as l,apiRawCall as p,loadViewHtml as c,setVizPort as m,setClientSupportsMcpApps as d,CLIENT_VERSION as u}from"./shared.js";import{registerDatasetsTool as f}from"./tools/datasets.js";import{registerJobsTool as g,JOBS_DESCRIPTION_BASE as _}from"./tools/jobs.js";import{registerResultsTool as b}from"./tools/results.js";import{registerExploreMapTool as h,RESULTS_EXPLORER_URI as y}from"./tools/explore_map.js";import{registerAccountTool as w}from"./tools/account.js";import{registerInferenceTool as j}from"./tools/inference.js";import{registerGuideBarsomTool as v}from"./tools/guide_barsom.js";import{registerTrainingGuidanceTool as P}from"./tools/training_guidance.js";import{registerFeedbackTool as x}from"./tools/feedback.js";import{registerTrainingPrepTools as I,TRAINING_PREP_URI as k}from"./tools/training_prep.js";import{registerTrainingMonitorTool as M,TRAINING_MONITOR_URI as O}from"./tools/training_monitor.js";import{resolvePrepareTrainingPromptText as S}from"./prepare_training_prompt.js";i||(console.error("Error: BARIVIA_API_KEY not set. Set it in your MCP client config."),process.exit(1));const A=new e({name:"analytics-engine",version:u,instructions:'# Barivia Mapping Analytics Engine\n\nSelf-organizing map (SOM) analytics: project high-dimensional data to a 2D grid for clusters, gradients, and anomalies.\n\n## Workflow (short)\n\nUpload (`datasets(upload)`) → `datasets(preview)` and `datasets(analyze)` before train → submit one of `jobs(train_map)`, `jobs(train_siom_map)`, or `jobs(train_floop_siom)` (only if plan allows FLooP) → poll `jobs(status)` every 10–15s until `completed` → `results(get)` for metrics and figures (there is no separate analyze tool). Then `jobs(compare)`, `results(download/recolor/transition_flow)`, or `inference` as needed.\n\n**Full detail:** Call `guide_barsom_workflow` for plan-scoped tool map, training modes, async rules, optional MCP App UIs, and step-by-step SOP (from the Barivia API when online).\n\n## Tool map (compact)\n\n| Area | Tool | Notes |\n|------|------|--------|\n| Data | `datasets` | upload, preview, analyze, list, subset, add_expression, reduce_spectral (pca/log_sample/uniform_sample/stats for long ordered numeric blocks), delete |\n| Jobs | `jobs` | train_map, train_impute (sparse data: map + dense imputed.csv), train_siom_map, train_floop_siom (entitled), status, list, compare, cancel, delete, batch_predict, run_baseline_study; `train_floop_chain` = deprecated alias for train_floop_siom |\n| Results | `results` | get (figures="none" for metrics-only), export, download, recolor (async), transition_flow (async; time-ordered rows only) |\n| Inference | `inference` | predict (regime-aware; output="compact"|"annotated"), impute_column (neighbor-pool fill for a non-training column), compare, project_columns, report |\n| Account | `account` | status, burst/compute actions, history, add_funds |\n| Bootstrap | `guide_barsom_workflow` | orientation + SOP |\n| Parameters | `training_guidance` | presets and field hints (API-scoped) |\n| Prep | `prepare_training` prompt, `training_prep` + `submit_prepared_training` | checklist / interactive UI |\n| Explore | `results_explorer`, `training_monitor` | optional MCP Apps; `explore_map` = deprecated alias of `results_explorer`; `jobs(status)` and `results(get)` suffice without them |\n| Other | `send_feedback` | only after user agrees |\n\n## Async pattern\n\n- **Manual poll:** Training submits return `job_id` immediately — poll `jobs(status)` every 10–15s. **Running is not failed**; large grids or FLooP-SIOM can take many minutes. `max_nodes` (FLooP) is a total node budget, not grid side length.\n- **Often auto-polled:** `inference` actions, `results(recolor)`, `results(transition_flow)` may wait in-proxy; if you get a `job_id`, poll `jobs(status)` the same way.\n\nCredits: jobs consume compute credits; check `account(status)` before big runs. Slow networks: users can raise `BARIVIA_FETCH_TIMEOUT_MS`.\n\n## Constraints\n\n- Prep ladder: `prepare_training` prompt = narrative checklist; `training_guidance` = structured hints; `training_prep` = UI + guarded submit. Do not guess tiers or FLooP entitlement.\n- `inference(predict)`: prefer `dataset_id` for batch and for SIOM/irregular maps; single-row `rows` uses a fast path that can fail on some topologies — retry with `dataset_id`. FLooP-SIOM: if predict jobs fail while grid SIOM works, capture errors + `job_id`.\n- Column names are case-sensitive — match `datasets(preview)`.\n- Default training path is numeric/cyclic/temporal; use explicit `categorical_features` for baseline categoricals. `predict` must match the model contract.\n- After `recolor`, `transition_flow`, or `project_columns`, use the **new** `job_id` returned for follow-up `results` if applicable.'});n(A,y,y,{mimeType:s},async()=>{const e=await c("results-explorer");return{contents:[{uri:y,mimeType:s,text:e??"<html><body>Results Explorer view not built yet. Run: npm run build:views</body></html>"}]}}),n(A,k,k,{mimeType:s},async()=>{const e=await c("training-prep");return{contents:[{uri:k,mimeType:s,text:e??"<html><body>Training Preparation view not built yet.</body></html>"}]}}),n(A,O,O,{mimeType:s},async()=>{const e=await c("training-monitor");return{contents:[{uri:O,mimeType:s,text:e??"<html><body>Training Monitor view not built yet.</body></html>"}]}}),v(A),h(A),I(A),M(A),f(A),g(A,_),b(A),w(A),j(A),P(A),x(A),A.prompt("info","Short orientation for the Barivia Mapping MCP. For full plan-scoped workflow, tool map, and SOP, the model should call guide_barsom_workflow. Use when the user asks what this MCP can do or how to get started.",{},()=>({messages:[{role:"user",content:{type:"text",text:["Give a concise, scannable answer (headers + bullets):","","**What it is:** MCP client to the Barivia mapping engine (2D SOM / SIOM / FLooP-SIOM when entitled) over HTTPS.","","**First step:** Call `guide_barsom_workflow` for plan-scoped bootstrap (full tool list, async rules, training modes, optional MCP Apps, SOP).","","**Core path:** `datasets(upload)` → `datasets(preview)` + `datasets(analyze)` → choose training action → poll `jobs(status)` every 10–15s until completed → `results(get)` (all main figures/metrics; no separate analyze tool).","",'**Key tools:** `datasets` (data; reduce_spectral for spectra/long blocks), `jobs` (train/poll/compare/…; train_map accepts an optional `label` for readable compare rows), `results` (get/download/export/recolor/transition_flow; figures="none" for metrics-only), `inference` (predict; impute_column for topology-neighbor pool fill; compare; project_columns; report), `account` (status/credits/queue).',"","**Prep help:** `prepare_training` prompt (checklist) · `training_guidance` (presets/JSON hints) · `training_prep` + `submit_prepared_training` (interactive UI).","","**Optional UI:** `results_explorer`, `training_monitor` — nice for browsing; not required if you use `results` + `jobs(status)`.","","**After training:** `jobs(compare)` across runs, `results(recolor)`, `inference(project_columns)` for variables not in training, `transition_flow` only if rows are time-ordered.","","**Rules:** Running ≠ failed. Column names must match `datasets(preview)` exactly. Do not call `_fetch_figure` from chat (host/UI only); use `results(get)` or `results_explorer`.","","Offer `send_feedback` only after asking the user."].join("\n")}}]})),A.prompt("prepare_training","Narrative pre-training checklist (prompt). Use after upload and before train. Content is tier-scoped from the API when online. Prep ladder: this prompt = story checklist; training_guidance tool = JSON presets/parameter hints; training_prep tool = interactive UI + submit_prepared_training.",{dataset_id:o.string().describe("Dataset ID to prepare for training")},async({dataset_id:e})=>({messages:[{role:"user",content:{type:"text",text:await S(e)}}]}));const T=new t;(async function(){try{const e=await a(l,p,c);m(e)}catch(e){process.env.BARIVIA_VIZ_PORT&&console.error("Barivia viz server failed to start:",e)}const e=A.server;e.oninitialized=()=>{const t=e.getClientCapabilities(),o=r(t);d(!!o?.mimeTypes?.includes(s))},await A.connect(T)})().catch(console.error);
+import{McpServer as e}from"@modelcontextprotocol/sdk/server/mcp.js";import{StdioServerTransport as t}from"@modelcontextprotocol/sdk/server/stdio.js";import{z as o}from"zod";import{getUiCapability as r,registerAppResource as n,RESOURCE_MIME_TYPE as s}from"@modelcontextprotocol/ext-apps/server";import{startVizServer as a}from"./viz-server.js";import{API_KEY as i,apiCall as l,apiRawCall as p,loadViewHtml as c,setVizPort as m,setClientSupportsMcpApps as d,CLIENT_VERSION as u}from"./shared.js";import{registerDatasetsTool as f}from"./tools/datasets.js";import{registerJobsTool as g,JOBS_DESCRIPTION_BASE as _}from"./tools/jobs.js";import{registerResultsTool as b}from"./tools/results.js";import{registerExploreMapTool as h,RESULTS_EXPLORER_URI as y}from"./tools/explore_map.js";import{registerAccountTool as w}from"./tools/account.js";import{registerInferenceTool as j}from"./tools/inference.js";import{registerGuideBarsomTool as v}from"./tools/guide_barsom.js";import{registerTrainingGuidanceTool as P}from"./tools/training_guidance.js";import{registerFeedbackTool as x}from"./tools/feedback.js";import{registerTrainingPrepTools as I,TRAINING_PREP_URI as k}from"./tools/training_prep.js";import{registerTrainingMonitorTool as T,TRAINING_MONITOR_URI as M}from"./tools/training_monitor.js";import{resolvePrepareTrainingPromptText as O}from"./prepare_training_prompt.js";i||(console.error("Error: BARIVIA_API_KEY not set. Set it in your MCP client config."),process.exit(1));const A=new e({name:"analytics-engine",version:u,instructions:'# Barivia Mapping Analytics Engine\n\nSelf-organizing map (SOM) analytics: project high-dimensional data to a 2D grid for clusters, gradients, and anomalies.\n\n## Workflow (short)\n\nUpload (`datasets(upload)`) → `datasets(preview)` and `datasets(analyze)` before train → submit one of `jobs(train_map)`, `jobs(train_siom_map)`, or `jobs(train_floop_siom)` (only if plan allows FLooP) → poll `jobs(status)` every 10–15s until `completed` → `results(get)` for metrics and figures (there is no separate analyze tool). Then `jobs(compare)`, `results(download/recolor/transition_flow)`, or `inference` as needed.\n\n**Full detail:** Call `guide_barsom_workflow` for plan-scoped tool map, training modes, async rules, optional MCP App UIs, and step-by-step SOP (from the Barivia API when online).\n\n## Tool map (compact)\n\n| Area | Tool | Notes |\n|------|------|--------|\n| Data | `datasets` | upload, preview, analyze, list, subset, add_expression, reduce_spectral (pca/log_sample/uniform_sample/stats for long ordered numeric blocks), delete |\n| Jobs | `jobs` | train_map, train_impute (sparse data: map + dense imputed.csv), train_siom_map, train_floop_siom (entitled), status, list, compare, cancel, delete, batch_predict, run_baseline_study; `train_floop_chain` = deprecated alias for train_floop_siom |\n| Results | `results` | get (figures="none" for metrics-only), export, download, recolor (async), transition_flow (async; time-ordered rows only) |\n| Inference | `inference` | predict (regime-aware; output="compact"|"annotated"), impute_column (neighbor-pool fill for a non-training column), compare, project_columns, report |\n| Account | `account` | status, burst/compute actions, history, add_funds |\n| Bootstrap | `guide_barsom_workflow` | orientation + SOP |\n| Parameters | `training_guidance` | presets and field hints (API-scoped) |\n| Prep | `prepare_training` prompt, `training_prep` + `submit_prepared_training` | checklist / interactive UI |\n| Explore | `results_explorer`, `training_monitor` | optional MCP Apps; `explore_map` = deprecated alias of `results_explorer`; `jobs(status)` and `results(get)` suffice without them |\n| Other | `send_feedback` | only after user agrees |\n\n## Async pattern\n\n- **Manual poll:** Training submits return `job_id` immediately — poll `jobs(status)` every 10–15s. **Running is not failed**; large grids or FLooP-SIOM can take many minutes. `max_nodes` (FLooP) is a total node budget, not grid side length.\n- **Often auto-polled:** `inference` actions, `results(recolor)`, `results(transition_flow)` may wait in-proxy; if you get a `job_id`, poll `jobs(status)` the same way.\n\nCredits: jobs consume compute credits; check `account(status)` before big runs. Per-request HTTP timeout defaults to 60s (`datasets(analyze)` is async + auto-polled, so it is not bound by it); on slow networks raise `BARIVIA_FETCH_TIMEOUT_MS`. Timeouts are NOT auto-retried (avoids duplicating slow/expensive work).\n\n## Constraints\n\n- Prep ladder: `prepare_training` prompt = narrative checklist; `training_guidance` = structured hints; `training_prep` = UI + guarded submit. Do not guess tiers or FLooP entitlement.\n- `inference(predict)`: prefer `dataset_id` for batch and for SIOM/irregular maps; single-row `rows` uses a fast path that can fail on some topologies — retry with `dataset_id`. FLooP-SIOM: if predict jobs fail while grid SIOM works, capture errors + `job_id`.\n- Column names are case-sensitive — match `datasets(preview)`.\n- Default training path is numeric/cyclic/temporal; use explicit `categorical_features` for baseline categoricals. `predict` must match the model contract.\n- After `recolor`, `transition_flow`, or `project_columns`, use the **new** `job_id` returned for follow-up `results` if applicable.'});n(A,y,y,{mimeType:s},async()=>{const e=await c("results-explorer");return{contents:[{uri:y,mimeType:s,text:e??"<html><body>Results Explorer view not built yet. Run: npm run build:views</body></html>"}]}}),n(A,k,k,{mimeType:s},async()=>{const e=await c("training-prep");return{contents:[{uri:k,mimeType:s,text:e??"<html><body>Training Preparation view not built yet.</body></html>"}]}}),n(A,M,M,{mimeType:s},async()=>{const e=await c("training-monitor");return{contents:[{uri:M,mimeType:s,text:e??"<html><body>Training Monitor view not built yet.</body></html>"}]}}),v(A),h(A),I(A),T(A),f(A),g(A,_),b(A),w(A),j(A),P(A),x(A),A.prompt("info","Short orientation for the Barivia Mapping MCP. For full plan-scoped workflow, tool map, and SOP, the model should call guide_barsom_workflow. Use when the user asks what this MCP can do or how to get started.",{},()=>({messages:[{role:"user",content:{type:"text",text:["Give a concise, scannable answer (headers + bullets):","","**What it is:** MCP client to the Barivia mapping engine (2D SOM / SIOM / FLooP-SIOM when entitled) over HTTPS.","","**First step:** Call `guide_barsom_workflow` for plan-scoped bootstrap (full tool list, async rules, training modes, optional MCP Apps, SOP).","","**Core path:** `datasets(upload)` → `datasets(preview)` + `datasets(analyze)` → choose training action → poll `jobs(status)` every 10–15s until completed → `results(get)` (all main figures/metrics; no separate analyze tool).","",'**Key tools:** `datasets` (data; reduce_spectral for spectra/long blocks), `jobs` (train/poll/compare/…; train_map accepts an optional `label` for readable compare rows), `results` (get/download/export/recolor/transition_flow; figures="none" for metrics-only), `inference` (predict; impute_column for topology-neighbor pool fill; compare; project_columns; report), `account` (status/credits/queue).',"","**Prep help:** `prepare_training` prompt (checklist) · `training_guidance` (presets/JSON hints) · `training_prep` + `submit_prepared_training` (interactive UI).","","**Optional UI:** `results_explorer`, `training_monitor` — nice for browsing; not required if you use `results` + `jobs(status)`.","","**After training:** `jobs(compare)` across runs, `results(recolor)`, `inference(project_columns)` for variables not in training, `transition_flow` only if rows are time-ordered.","","**Rules:** Running ≠ failed. Column names must match `datasets(preview)` exactly. Do not call `_fetch_figure` from chat (host/UI only); use `results(get)` or `results_explorer`.","","Offer `send_feedback` only after asking the user."].join("\n")}}]})),A.prompt("prepare_training","Narrative pre-training checklist (prompt). Use after upload and before train. Content is tier-scoped from the API when online. Prep ladder: this prompt = story checklist; training_guidance tool = JSON presets/parameter hints; training_prep tool = interactive UI + submit_prepared_training.",{dataset_id:o.string().describe("Dataset ID to prepare for training")},async({dataset_id:e})=>({messages:[{role:"user",content:{type:"text",text:await O(e)}}]}));const S=new t;(async function(){try{const e=await a(l,p,c);m(e)}catch(e){process.env.BARIVIA_VIZ_PORT&&console.error("Barivia viz server failed to start:",e)}const e=A.server;e.oninitialized=()=>{const t=e.getClientCapabilities(),o=r(t);d(!!o?.mimeTypes?.includes(s))},await A.connect(S)})().catch(console.error);

package/dist/shared.js

@@ -2,6 +2,10 @@
  * Shared configuration, helpers, types, and state for the MCP proxy.
  */
 import fs from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import { createGzip } from "node:zlib";
+import { createHash } from "node:crypto";
+import { Readable } from "node:stream";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { logInfo } from "./logger.js";
@@ -12,7 +16,7 @@ export const API_URL = process.env.BARIVIA_API_URL ??
     process.env.BARSOM_API_URL ??
     "https://api.barivia.se";
 export const API_KEY = process.env.BARIVIA_API_KEY ?? process.env.BARSOM_API_KEY ?? "";
-export const FETCH_TIMEOUT_MS = parseInt(process.env.BARIVIA_FETCH_TIMEOUT_MS ?? "30000", 10);
+export const FETCH_TIMEOUT_MS = parseInt(process.env.BARIVIA_FETCH_TIMEOUT_MS ?? "60000", 10);
 export const MAX_RETRIES = 2;
 export const RETRYABLE_STATUS = new Set([502, 503, 504]);
 /**
@@ -20,7 +24,7 @@ export const RETRYABLE_STATUS = new Set([502, 503, 504]);
  * X-Barsom-Client-Version so the server can annotate tool guidance with the
  * wrapper version each action requires. Keep in sync with package.json on bump.
  */
-export const CLIENT_VERSION = "0.10.5";
+export const CLIENT_VERSION = "0.11.1";
 /** User-facing links; keep aligned with barivia.se / api.barivia.se. */
 export const PUBLIC_SITE_ORIGIN = "https://barivia.se";
 /** Self-serve account dashboard (manage plan, billing, and API keys). */
@@ -29,6 +33,12 @@ export const PUBLIC_DASHBOARD_URL = `${PUBLIC_SITE_ORIGIN}/dashboard`;
 export const POLL_DERIVE_MAX_MS = 120_000;
 /** Large CSV uploads may exceed default FETCH_TIMEOUT_MS. */
 export const UPLOAD_DATASET_TIMEOUT_MS = 180_000;
+/** Files at/above this size use the presigned direct-to-R2 streaming upload path. */
+export const LARGE_UPLOAD_BYTES = 64 * 1024 * 1024; // 64 MB
+/** Timeout for a direct presigned PUT of a large (gzipped) file to R2. */
+export const PRESIGNED_PUT_TIMEOUT_MS = 30 * 60_000; // 30 min
+/** Poll window for the async stage_dataset job (large files take minutes). */
+export const POLL_STAGE_MAX_MS = 30 * 60_000; // 30 min
 // ---------------------------------------------------------------------------
 // Shared state (mutable)
 // ---------------------------------------------------------------------------
@@ -52,12 +62,51 @@ export function setClientSupportsMcpApps(value) {
 export function isTransientError(err, status) {
     if (status !== undefined && RETRYABLE_STATUS.has(status))
         return true;
-    if (err instanceof DOMException && err.name === "AbortError")
-        return true;
+    // NOTE: a client-side timeout (AbortError) is deliberately NOT retried. Re-firing
+    // a slow/expensive request (e.g. analyze, large upload) while the server is still
+    // processing the first one only multiplies load and risks duplicates — the user
+    // should raise BARIVIA_FETCH_TIMEOUT_MS or use the async (job_id + poll) path.
     if (err instanceof TypeError)
         return true; // network-level fetch failure
     return false;
 }
+/** Streaming SHA-256 of a file (for idempotency keys) without reading it into memory. */
+export async function streamFileSha256(srcPath) {
+    return new Promise((resolve, reject) => {
+        const h = createHash("sha256");
+        const s = createReadStream(srcPath);
+        s.on("data", (chunk) => h.update(chunk));
+        s.on("end", () => resolve(h.digest("hex")));
+        s.on("error", reject);
+    });
+}
+/**
+ * Stream a local file through gzip directly to a presigned PUT URL (e.g. R2),
+ * never materializing the file in memory. Bypasses the API/Cloudflare body cap.
+ */
+export async function putPresignedStream(url, srcPath, contentType, timeoutMs = PRESIGNED_PUT_TIMEOUT_MS) {
+    const gz = createReadStream(srcPath).pipe(createGzip());
+    const webStream = Readable.toWeb(gz);
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const resp = await fetch(url, {
+            method: "PUT",
+            body: webStream,
+            headers: { "Content-Type": contentType },
+            // Required by Node's fetch to send a streaming request body.
+            duplex: "half",
+            signal: controller.signal,
+        });
+        if (!resp.ok) {
+            const t = await resp.text().catch(() => "");
+            throw new Error(`Presigned upload failed: HTTP ${resp.status} ${t.slice(0, 200)}`);
+        }
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
 export async function fetchWithTimeout(url, init, timeoutMs = FETCH_TIMEOUT_MS) {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), timeoutMs);

package/dist/tools/datasets.js

@@ -1,9 +1,97 @@
 import path from "node:path";
 import fs from "node:fs/promises";
 import { gzipSync } from "node:zlib";
+import { createHash } from "node:crypto";
 import { z } from "zod";
 import { registerAuditedTool } from "../audit.js";
-import { apiCall, getWorkspaceRootAsync, resolveFilePathForUpload, textResult, pollUntilComplete, POLL_DERIVE_MAX_MS, UPLOAD_DATASET_TIMEOUT_MS, } from "../shared.js";
+import { apiCall, getWorkspaceRootAsync, resolveFilePathForUpload, textResult, pollUntilComplete, POLL_DERIVE_MAX_MS, UPLOAD_DATASET_TIMEOUT_MS, LARGE_UPLOAD_BYTES, PRESIGNED_PUT_TIMEOUT_MS, POLL_STAGE_MAX_MS, streamFileSha256, putPresignedStream, } from "../shared.js";
+/**
+ * Format a dataset analyze result (from either the async job summary or the
+ * legacy sync GET response) into the human-readable text block.
+ */
+function formatAnalyzeResult(data, dataset_id) {
+    const totalRows = (data.total_rows ?? data.total_data_rows);
+    const analyzedRows = data.analyzed_rows;
+    const sampled = data.sampled === true;
+    const sampleNote = sampled && analyzedRows !== undefined && totalRows !== undefined
+        ? `Note: analyzed a ${analyzedRows.toLocaleString()}-row sample of ${totalRows.toLocaleString()} rows (subsampled for speed — correlations use an even spread across the table, periodicity uses the first rows).`
+        : "";
+    const note = data.note;
+    if (note) {
+        return [
+            `Dataset: ${data.name} (${data.dataset_id ?? dataset_id})`,
+            `${totalRows ?? "?"} rows.`,
+            sampleNote,
+            ``,
+            note,
+        ]
+            .filter(Boolean)
+            .join("\n");
+    }
+    const columns = data.columns ?? [];
+    const highPairs = data.high_correlation_pairs ?? [];
+    const uniqueness = data.uniqueness_scores ?? {};
+    const periodicity = data.periodicity ?? [];
+    const rec = data.recommendations ?? { train: [], consider_dropping: [], project_later: [], low_variance: [] };
+    const trainList = rec.train ?? [];
+    const considerDropping = rec.consider_dropping ?? [];
+    const projectLater = rec.project_later ?? [];
+    const lowVariance = rec.low_variance ?? [];
+    const periodicityCaveat = data.periodicity_caveat;
+    const lines = [
+        `Pre-training analysis: ${data.name} (${data.dataset_id ?? dataset_id})`,
+        `${totalRows ?? "?"} rows × ${columns.length} numeric columns analyzed`,
+        ...(sampleNote ? [sampleNote] : []),
+        ``,
+        ...(periodicityCaveat ? [`Note: ${periodicityCaveat}`, ``] : []),
+        `Column recommendations:`,
+        `  Train (use in jobs(action=train_map)): ${trainList.length ? trainList.join(", ") : "—"}`,
+        `  Consider dropping (highly correlated with another): ${considerDropping.length ? considerDropping.join(", ") : "—"}`,
+        `  Project later (after training, use inference(action=project_columns, job_id=<training_job_id>, dataset_id=<dataset_id>, columns=[...]) onto map): ${projectLater.length ? projectLater.join(", ") : "—"}`,
+        `  Low variance (near-constant): ${lowVariance.length ? lowVariance.join(", ") : "—"}`,
+        ``,
+    ];
+    if (highPairs.length > 0) {
+        lines.push(`High correlation pairs (|r| > 0.85):`);
+        for (const p of highPairs) {
+            lines.push(`  • ${p.column_a} ↔ ${p.column_b}: r = ${p.correlation}`);
+        }
+        lines.push(``);
+    }
+    if (Object.keys(uniqueness).length > 0) {
+        lines.push(`Uniqueness score (1 − max|r| with others; higher = more unique):`);
+        for (const col of columns) {
+            const u = uniqueness[col];
+            if (u !== undefined)
+                lines.push(`  • ${col}: ${u.toFixed(3)}`);
+        }
+        lines.push(``);
+    }
+    if (periodicity.length > 0) {
+        lines.push(`Periodicity ranking (by autocorrelation strength at lags 7, 12, 24, 52, 365):`);
+        for (const p of periodicity) {
+            lines.push(`  • ${p.column}: dominant lag ${p.dominant_lag} (score ${p.dominant_score})`);
+        }
+        lines.push(``);
+    }
+    const nextSteps = [];
+    if (trainList.length > 0) {
+        nextSteps.push(`Train with columns: [${trainList.join(", ")}].`);
+    }
+    if (projectLater.length > 0) {
+        nextSteps.push(`After training, use inference(action=project_columns, job_id=<training_job_id>, dataset_id=${dataset_id}, columns=[${projectLater.map((c) => `"${c}"`).join(", ")}]) to project these onto the map.`);
+    }
+    const periodicStrong = periodicity.filter((p) => p.dominant_score > 0.4);
+    if (periodicStrong.length > 0) {
+        const lags = [...new Set(periodicStrong.map((p) => p.dominant_lag))].sort((a, b) => a - b);
+        nextSteps.push(`Note: periodicity is computed on row order. If your CSV is not sorted by time, ignore lag-based suggestions; only use cyclic_features when the column is genuinely cyclic (e.g. hour, month, angle).`);
+        nextSteps.push(`Columns show periodic behavior at lags ${lags.join(", ")} (assuming row order is meaningful). Consider cyclic_features or temporal_features with matching periods.`);
+        nextSteps.push(`For datetime columns: run datasets(action=preview) for temporal_suggestions; match lags (e.g. 365→day_of_year, 12→month, 7→day_of_week, 24→hour_of_day) to choose which temporal components to extract.`);
+    }
+    nextSteps.push(`Then call jobs(action=train_map, dataset_id=${dataset_id}, columns=[...], ...).`);
+    lines.push(`Next steps:`, ...nextSteps.map((s) => `  ${s}`));
+    return lines.join("\n");
+}
 export function registerDatasetsTool(server) {
     registerAuditedTool(server, "datasets", `Manage datasets: upload, preview, list, subset, add_expression, or delete.
 
@@ -26,12 +114,13 @@ Step 1 (after upload): Upload, then always datasets(action=preview) to verify co
 action=preview: Show columns, stats, sample rows, cyclic/datetime detections. ALWAYS preview before jobs(action=train_map) on an unfamiliar dataset.
 action=analyze: Pre-training analysis on numeric columns — Pearson correlation, autocorrelation periodicity scores, and column recommendations (train, consider_dropping, project_later, low_variance). Use to choose which columns to include in training and which to project onto the map after training.
 action=list: List all datasets belonging to the organisation with id, name, rows, cols, created_at (use created_at or id to distinguish datasets with the same name).
-action=subset: Create a new dataset from a subset of an existing one. Requires name and at least one of row_range or filters.
+action=subset: Create a new dataset from a subset of an existing one. Requires name and at least one of row_range, filters, or sample_n.
   - row_range: [start, end] 1-based inclusive (e.g. [1, 2000] for first 2000 rows)
   - filters: array of conditions, ALL must match (AND logic). Each: { column, op, value }.
     Operators: eq, ne, in, gt, lt, gte, lte, between
     Examples: { column: "region", op: "eq", value: "Europe" } | { column: "age", op: "between", value: [18, 65] }
-  - Combine row_range + filters to slice both rows and values.
+  - sample_n: keep a random N-row sample (seeded via sample_seed, default 42; row order preserved). Shrink a huge table server-side instead of pre-sampling locally. Applied after row_range/filters; can also be used alone.
+  - Combine row_range + filters + sample_n to slice rows, values, then downsample.
   - Single filter object is also accepted (auto-wrapped).
 action=reduce_spectral: Run a pre-training reducer over an ordered block of numeric columns. All four methods produce one feature vector per row (rows in = rows out; only the column dimension is collapsed) and append derived columns to the dataset. Choose by data shape:
   - pca:            top-k principal components — general first try when many columns are correlated (spectroscopy, gene panels, sensor arrays). Returns explained_variance_ratio.
@@ -104,7 +193,18 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
             .min(1)
             .optional()
             .describe("action=reduce_spectral: output dimensionality. Required for pca/log_sample/uniform_sample; ignored for stats."),
-    }, async ({ action, name, file_path, csv_data, dataset_id, n_rows, row_range, filters, filter, expression, options, method, columns_block, k }) => {
+        sample_n: z
+            .number()
+            .int()
+            .min(1)
+            .optional()
+            .describe("action=subset: keep a random N-row sample (seeded, row order preserved). Use to shrink a huge table server-side instead of pre-sampling locally. Combine with row_range/filters, or use alone."),
+        sample_seed: z
+            .number()
+            .int()
+            .optional()
+            .describe("action=subset: RNG seed for sample_n (default 42; same seed + dataset reproduces the same sample)."),
+    }, async ({ action, name, file_path, csv_data, dataset_id, n_rows, row_range, filters, filter, expression, options, method, columns_block, k, sample_n, sample_seed }) => {
         if (action === "upload") {
             if (!name)
                 throw new Error("datasets(upload) requires name");
@@ -116,20 +216,46 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
                 if (ext !== ".csv" && ext !== ".tsv") {
                     throw new Error("Only .csv and .tsv files can be uploaded as datasets.");
                 }
-                const MAX_UPLOAD_BYTES = 256 * 1024 * 1024; // 256 MB (gzip keeps the wire payload small)
+                const HARD_MAX_BYTES = 5 * 1024 * 1024 * 1024; // 5 GB (R2 single-PUT limit)
+                let stat;
                 try {
-                    const stat = await fs.stat(resolved);
-                    if (stat.size > MAX_UPLOAD_BYTES) {
-                        throw new Error(`File too large (${(stat.size / 1024 / 1024).toFixed(1)} MB). Maximum upload size is ${MAX_UPLOAD_BYTES / 1024 / 1024} MB.`);
-                    }
-                    body = await fs.readFile(resolved, "utf-8");
+                    stat = await fs.stat(resolved);
                 }
-                catch (err) {
-                    if (err instanceof Error && err.message.includes("too large"))
-                        throw err;
+                catch {
                     throw new Error(`File not accessible at resolved path. file_path is relative to workspace root. ` +
                         `Set BARIVIA_WORKSPACE_ROOT in your MCP config env if needed (current: ${await getWorkspaceRootAsync(server)}).`);
                 }
+                if (stat.size > HARD_MAX_BYTES) {
+                    throw new Error(`File too large (${(stat.size / 1024 / 1024 / 1024).toFixed(2)} GB). Maximum upload size is 5 GB.`);
+                }
+                // Large files: stream gzip directly to R2 (presigned PUT), then stage async.
+                // Never reads the file into memory or sends it through the API/Cloudflare.
+                if (stat.size >= LARGE_UPLOAD_BYTES) {
+                    const idem = await streamFileSha256(resolved);
+                    const init = (await apiCall("POST", "/v1/datasets/upload-url", { name, size_bytes: stat.size }, { "Idempotency-Key": idem }));
+                    const datasetId = (init.dataset_id ?? init.id);
+                    if (init.idempotent_replay) {
+                        return textResult({ id: datasetId, status: init.status, idempotent_replay: true,
+                            suggested_next_step: `datasets(action=preview, dataset_id=${datasetId})` });
+                    }
+                    await putPresignedStream(init.upload_url, resolved, init.content_type ?? "application/octet-stream", PRESIGNED_PUT_TIMEOUT_MS);
+                    const fin = (await apiCall("POST", `/v1/datasets/${datasetId}/finalize`, {}));
+                    const jobId = (fin.id ?? fin.job_id);
+                    const poll = await pollUntilComplete(jobId, POLL_STAGE_MAX_MS);
+                    if (poll.status === "failed") {
+                        return textResult({ id: datasetId, status: "failed", error: poll.error ?? "staging failed" });
+                    }
+                    const ready = poll.status === "completed";
+                    return textResult({
+                        id: datasetId,
+                        status: ready ? "ready" : "staging",
+                        job_id: jobId,
+                        suggested_next_step: ready
+                            ? `datasets(action=preview, dataset_id=${datasetId}) to inspect columns before training.`
+                            : `Still staging; poll jobs(action=status, job_id="${jobId}") then datasets(action=preview, dataset_id=${datasetId}).`,
+                    });
+                }
+                body = await fs.readFile(resolved, "utf-8");
             }
             else if (csv_data && csv_data.length > 0) {
                 body = csv_data;
@@ -143,6 +269,9 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
             const uploadHeaders = {
                 "X-Dataset-Name": name,
                 "Content-Type": "text/csv",
+                // Deterministic key so a timed-out retry of the SAME upload reconciles to
+                // the original dataset server-side instead of creating a duplicate.
+                "Idempotency-Key": createHash("sha256").update(`${name}\n`).update(body).digest("hex"),
             };
             let uploadBody = body;
             if (Buffer.byteLength(body, "utf-8") > GZIP_THRESHOLD) {
@@ -243,76 +372,34 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
         if (action === "analyze") {
             if (!dataset_id)
                 throw new Error("datasets(analyze) requires dataset_id");
-            const data = (await apiCall("GET", `/v1/datasets/${dataset_id}/analyze`));
-            const note = data.note;
-            if (note) {
-                return { content: [{ type: "text", text: `Dataset: ${data.name} (${data.dataset_id})\n${data.total_rows} rows.\n\n${note}` }] };
-            }
-            const columns = data.columns ?? [];
-            const highPairs = data.high_correlation_pairs ?? [];
-            const uniqueness = data.uniqueness_scores ?? {};
-            const periodicity = data.periodicity ?? [];
-            const rec = data.recommendations ?? { train: [], consider_dropping: [], project_later: [], low_variance: [] };
-            const trainList = rec.train ?? [];
-            const considerDropping = rec.consider_dropping ?? [];
-            const projectLater = rec.project_later ?? [];
-            const lowVariance = rec.low_variance ?? [];
-            const periodicityCaveat = data.periodicity_caveat;
-            const lines = [
-                `Pre-training analysis: ${data.name} (${data.dataset_id})`,
-                `${data.total_rows} rows × ${columns.length} numeric columns analyzed`,
-                ``,
-                ...(periodicityCaveat ? [`Note: ${periodicityCaveat}`, ``] : []),
-                `Column recommendations:`,
-                `  Train (use in jobs(action=train_map)): ${trainList.length ? trainList.join(", ") : "—"}`,
-                `  Consider dropping (highly correlated with another): ${considerDropping.length ? considerDropping.join(", ") : "—"}`,
-                `  Project later (after training, use inference(action=project_columns, job_id=<training_job_id>, dataset_id=<dataset_id>, columns=[...]) onto map): ${projectLater.length ? projectLater.join(", ") : "—"}`,
-                `  Low variance (near-constant): ${lowVariance.length ? lowVariance.join(", ") : "—"}`,
-                ``,
-            ];
-            if (highPairs.length > 0) {
-                lines.push(`High correlation pairs (|r| > 0.85):`);
-                for (const p of highPairs) {
-                    lines.push(`  • ${p.column_a} ↔ ${p.column_b}: r = ${p.correlation}`);
+            // Async path: enqueue an analyze job, poll, then fetch results. The proxy
+            // auto-polls so the caller still issues a single tool call. Falls back to
+            // the legacy synchronous GET on older APIs that lack the POST route (404).
+            let data;
+            try {
+                const submit = (await apiCall("POST", `/v1/datasets/${dataset_id}/analyze`));
+                const jobId = (submit.id ?? submit.job_id);
+                if (!jobId)
+                    throw new Error("analyze did not return a job id");
+                const poll = await pollUntilComplete(jobId, POLL_DERIVE_MAX_MS);
+                if (poll.status === "failed") {
+                    return { content: [{ type: "text", text: `datasets(analyze) job ${jobId} failed: ${poll.error ?? "unknown error"}` }] };
                 }
-                lines.push(``);
-            }
-            if (Object.keys(uniqueness).length > 0) {
-                lines.push(`Uniqueness score (1 − max|r| with others; higher = more unique):`);
-                for (const col of columns) {
-                    const u = uniqueness[col];
-                    if (u !== undefined)
-                        lines.push(`  • ${col}: ${u.toFixed(3)}`);
+                if (poll.status !== "completed") {
+                    return { content: [{ type: "text", text: `datasets(analyze) job ${jobId} is still running. Poll with jobs(action=status, job_id="${jobId}") then results(action=get, job_id="${jobId}").` }] };
                 }
-                lines.push(``);
+                const results = (await apiCall("GET", `/v1/results/${jobId}`));
+                data = (results.summary ?? results);
             }
-            if (periodicity.length > 0) {
-                lines.push(`Periodicity ranking (by autocorrelation strength at lags 7, 12, 24, 52, 365):`);
-                for (const p of periodicity) {
-                    const col = p.column;
-                    const dominantLag = p.dominant_lag;
-                    const score = p.dominant_score;
-                    lines.push(`  • ${col}: dominant lag ${dominantLag} (score ${score})`);
+            catch (err) {
+                if (err?.httpStatus === 404) {
+                    data = (await apiCall("GET", `/v1/datasets/${dataset_id}/analyze`));
+                }
+                else {
+                    throw err;
                 }
-                lines.push(``);
-            }
-            const nextSteps = [];
-            if (trainList.length > 0) {
-                nextSteps.push(`Train with columns: [${trainList.join(", ")}].`);
-            }
-            if (projectLater.length > 0) {
-                nextSteps.push(`After training, use inference(action=project_columns, job_id=<training_job_id>, dataset_id=${dataset_id}, columns=[${projectLater.map((c) => `"${c}"`).join(", ")}]) to project these onto the map.`);
-            }
-            const periodicStrong = periodicity.filter((p) => p.dominant_score > 0.4);
-            if (periodicStrong.length > 0) {
-                const lags = [...new Set(periodicStrong.map((p) => p.dominant_lag))].sort((a, b) => a - b);
-                nextSteps.push(`Note: periodicity is computed on row order. If your CSV is not sorted by time, ignore lag-based suggestions; only use cyclic_features when the column is genuinely cyclic (e.g. hour, month, angle).`);
-                nextSteps.push(`Columns show periodic behavior at lags ${lags.join(", ")} (assuming row order is meaningful). Consider cyclic_features or temporal_features with matching periods.`);
-                nextSteps.push(`For datetime columns: run datasets(action=preview) for temporal_suggestions; match lags (e.g. 365→day_of_year, 12→month, 7→day_of_week, 24→hour_of_day) to choose which temporal components to extract.`);
             }
-            nextSteps.push(`Then call jobs(action=train_map, dataset_id=${dataset_id}, columns=[...], ...).`);
-            lines.push(`Next steps:`, ...nextSteps.map((s) => `  ${s}`));
-            return { content: [{ type: "text", text: lines.join("\n") }] };
+            return { content: [{ type: "text", text: formatAnalyzeResult(data, dataset_id) }] };
         }
         if (action === "add_expression") {
             if (!dataset_id)
@@ -411,14 +498,18 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
             if (!name)
                 throw new Error("datasets(subset) requires name");
             const allFilters = filters ?? (filter ? [filter] : undefined);
-            if (row_range === undefined && allFilters === undefined) {
-                throw new Error("datasets(subset) requires at least one of row_range or filters");
+            if (row_range === undefined && allFilters === undefined && sample_n === undefined) {
+                throw new Error("datasets(subset) requires at least one of row_range, filters, or sample_n");
             }
             const body = { name };
             if (row_range !== undefined)
                 body.row_range = row_range;
             if (allFilters !== undefined)
                 body.filters = allFilters;
+            if (sample_n !== undefined)
+                body.sample_n = sample_n;
+            if (sample_seed !== undefined)
+                body.sample_seed = sample_seed;
             const data = await apiCall("POST", `/v1/datasets/${dataset_id}/subset`, body);
             return textResult(data);
         }

package/dist/tools/feedback.js

@@ -2,9 +2,21 @@ import { z } from "zod";
 import { registerAuditedTool } from "../audit.js";
 import { apiCall, API_URL, fetchWithTimeout, textResult } from "../shared.js";
 export function registerFeedbackTool(server) {
-    registerAuditedTool(server, "send_feedback", `Send feedback or feature requests to Barivia developers (max 1400 characters, ~190 words). Use when the user has suggestions, ran into issues, or wants something improved. Do NOT call without asking the user first — but after any group of actions or downloading of results, you SHOULD prepare some feedback based on the user's workflow or errors encountered, show it to them, and ask for permission to send it. Once they accept, call this tool.`, { feedback: z.string().max(1400).describe("Feedback text (max 1400 characters)") }, async ({ feedback }) => {
+    registerAuditedTool(server, "send_feedback", `Send feedback or feature requests to Barivia developers. Use when the user has suggestions, ran into issues, or wants something improved. Do NOT call without asking the user first — but after any group of actions or downloading of results, you SHOULD prepare feedback based on the user's workflow or errors encountered, show it to them, and ask for permission to send it. Once they accept, call this tool.
+
+For a substantial issue, prefer feedback_items: submit several focused instances (each max 1400 chars) covering, e.g., symptoms, exact reproduction steps, environment, and concrete asks — they are stored together as one batch so developers see the full picture. Use the single feedback field for a short one-off note.`, {
+        feedback: z.string().max(1400).optional().describe("Single feedback note (max 1400 characters). Use feedback_items instead for a multi-part report."),
+        feedback_items: z.array(z.string().max(1400)).max(10).optional().describe("Several feedback instances (each max 1400 characters, up to 10), stored together as one batch. Prefer this for a detailed issue: split it into focused parts (symptoms, reproduction, environment, asks)."),
+    }, async ({ feedback, feedback_items }) => {
+        const items = (feedback_items ?? []).map((s) => s.trim()).filter((s) => s.length > 0);
+        if (feedback && feedback.trim().length > 0)
+            items.unshift(feedback.trim());
+        if (items.length === 0) {
+            throw new Error("send_feedback requires feedback or feedback_items (at least one non-empty entry).");
+        }
+        const body = items.length === 1 ? { feedback: items[0] } : { feedback_items: items };
         try {
-            const data = await apiCall("POST", "/v1/feedback", { feedback });
+            const data = await apiCall("POST", "/v1/feedback", body);
             return textResult(data);
         }
         catch (err) {
@@ -14,7 +26,7 @@ export function registerFeedbackTool(server) {
                     const resp = await fetchWithTimeout(url, {
                         method: "POST",
                         headers: { "Content-Type": "application/json" },
-                        body: JSON.stringify({ feedback }),
+                        body: JSON.stringify(body),
                     });
                     const text = await resp.text();
                     if (resp.ok) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@barivia/barsom-mcp",
-  "version": "0.10.5",
+  "version": "0.11.1",
   "description": "barSOM MCP proxy — connect any MCP client to the barSOM cloud API for Self-Organizing Map analytics",
   "keywords": [
     "mcp",