@barivia/barsom-mcp 0.14.0 → 0.15.0

package/README.md

@@ -86,7 +86,7 @@ Call at the **start of mapping work** (or when the user asks what the MCP can do
 
 | Action | Use when |
 |--------|----------|
-| `upload` | Adding a new CSV — returns dataset_id |
+| `upload` | Adding a new CSV — returns dataset_id. Accepts `.csv`/`.tsv` **and pre-gzipped `.csv.gz`/`.tsv.gz`** (compress big tables locally to transfer ~3× less). Large files stream directly to object storage; multi-GB uploads are supported. |
 | `preview` | Before training — inspect columns, stats, cyclic/datetime hints |
 | `analyze` | Pre-training recommendations (correlation, columns to consider dropping, etc.) |
 | `list` | Finding dataset IDs |

package/dist/shared.js

@@ -2,10 +2,12 @@
  * Shared configuration, helpers, types, and state for the MCP proxy.
  */
 import fs from "node:fs/promises";
-import { createReadStream } from "node:fs";
+import { createReadStream, createWriteStream } from "node:fs";
 import { createGzip } from "node:zlib";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { logInfo } from "./logger.js";
@@ -24,7 +26,7 @@ export const RETRYABLE_STATUS = new Set([502, 503, 504]);
  * X-Barsom-Client-Version so the server can annotate tool guidance with the
  * wrapper version each action requires. Keep in sync with package.json on bump.
  */
-export const CLIENT_VERSION = "0.14.0";
+export const CLIENT_VERSION = "0.15.0";
 /** User-facing links; keep aligned with barivia.se / api.barivia.se. */
 export const PUBLIC_SITE_ORIGIN = "https://barivia.se";
 /** Self-serve account dashboard (manage plan, billing, and API keys). */
@@ -80,31 +82,64 @@ export async function streamFileSha256(srcPath) {
         s.on("error", reject);
     });
 }
+/** Turn a raw S3/R2 presigned-PUT error into a clean, actionable message. */
+function presignedPutError(status, bodyText) {
+    const snippet = bodyText.slice(0, 200);
+    if (status === 411 || /MissingContentLength/i.test(bodyText)) {
+        // Should no longer happen now that we always send Content-Length, but keep a
+        // clear message instead of leaking raw S3 XML if a CDN strips the header.
+        return new Error("Upload rejected by storage: the Content-Length header was missing. " +
+            "This is a client bug — please update @barivia/barsom-mcp to the latest version.");
+    }
+    if (status === 413 || /EntityTooLarge|entity is too large/i.test(bodyText)) {
+        return new Error("Upload rejected by storage: file exceeds the maximum upload size.");
+    }
+    if (status === 403) {
+        return new Error("Upload rejected by storage: the presigned URL expired or was invalid; retry datasets(action=upload).");
+    }
+    return new Error(`Presigned upload failed: HTTP ${status} ${snippet}`);
+}
 /**
- * Stream a local file through gzip directly to a presigned PUT URL (e.g. R2),
- * never materializing the file in memory. Bypasses the API/Cloudflare body cap.
+ * Stream a local file directly to a presigned PUT URL (e.g. R2), gzip-compressing
+ * on the way unless the source is already gzipped. Never materializes the payload
+ * in process memory: when compression is needed we gzip to a temp file first, then
+ * PUT that file with an explicit Content-Length.
+ *
+ * R2/S3 presigned PUT requires Content-Length and rejects chunked transfer-encoding
+ * with HTTP 411 (MissingContentLength). Node's fetch sends a streaming body as
+ * chunked unless an explicit Content-Length is set, and gzip output length is not
+ * known until compression finishes — hence the gzip-to-temp-file step.
  */
-export async function putPresignedStream(url, srcPath, contentType, timeoutMs = PRESIGNED_PUT_TIMEOUT_MS) {
-    const gz = createReadStream(srcPath).pipe(createGzip());
-    const webStream = Readable.toWeb(gz);
+export async function putPresignedStream(url, srcPath, contentType, timeoutMs = PRESIGNED_PUT_TIMEOUT_MS, alreadyGzipped = false) {
+    let putPath = srcPath;
+    let tempPath;
+    if (!alreadyGzipped) {
+        tempPath = path.join(os.tmpdir(), `barsom-upload-${randomUUID()}.csv.gz`);
+        await pipeline(createReadStream(srcPath), createGzip(), createWriteStream(tempPath));
+        putPath = tempPath;
+    }
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), timeoutMs);
     try {
+        const contentLength = (await fs.stat(putPath)).size;
+        const webStream = Readable.toWeb(createReadStream(putPath));
         const resp = await fetch(url, {
             method: "PUT",
             body: webStream,
-            headers: { "Content-Type": contentType },
+            headers: { "Content-Type": contentType, "Content-Length": String(contentLength) },
             // Required by Node's fetch to send a streaming request body.
             duplex: "half",
             signal: controller.signal,
         });
         if (!resp.ok) {
             const t = await resp.text().catch(() => "");
-            throw new Error(`Presigned upload failed: HTTP ${resp.status} ${t.slice(0, 200)}`);
+            throw presignedPutError(resp.status, t);
         }
     }
     finally {
         clearTimeout(timer);
+        if (tempPath)
+            await fs.rm(tempPath, { force: true }).catch(() => { });
     }
 }
 export async function fetchWithTimeout(url, init, timeoutMs = FETCH_TIMEOUT_MS) {

package/dist/tools/datasets.js

@@ -224,9 +224,14 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
             if (file_path) {
                 await apiCall("GET", "/v1/system/info");
                 const resolved = await resolveFilePathForUpload(file_path, server);
-                const ext = path.extname(resolved).toLowerCase();
-                if (ext !== ".csv" && ext !== ".tsv") {
-                    throw new Error("Only .csv and .tsv files can be uploaded as datasets.");
+                const lower = resolved.toLowerCase();
+                // Accept pre-gzipped CSV/TSV (.csv.gz / .tsv.gz) so large tables transfer
+                // ~3x smaller. Already-gzipped files are stored as-is (the platform stores
+                // every dataset gzipped anyway); plain files are gzipped on the way up.
+                const isGzipInput = lower.endsWith(".gz");
+                const baseExt = path.extname(isGzipInput ? lower.slice(0, -3) : lower);
+                if (baseExt !== ".csv" && baseExt !== ".tsv") {
+                    throw new Error("Only .csv, .tsv, .csv.gz, or .tsv.gz files can be uploaded as datasets.");
                 }
                 const HARD_MAX_BYTES = 5 * 1024 * 1024 * 1024; // 5 GB (R2 single-PUT limit)
                 let stat;
@@ -246,11 +251,14 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
                     const idem = await streamFileSha256(resolved);
                     const init = (await apiCall("POST", "/v1/datasets/upload-url", { name, size_bytes: stat.size }, { "Idempotency-Key": idem }));
                     const datasetId = (init.dataset_id ?? init.id);
-                    if (init.idempotent_replay) {
+                    // Only short-circuit when the prior upload actually completed. A stale
+                    // `pending_upload` row (a previous PUT that never landed) comes back with
+                    // a fresh upload_url and no idempotent_replay flag, so we re-PUT below.
+                    if (init.idempotent_replay && !init.upload_url) {
                         return textResult({ id: datasetId, status: init.status, idempotent_replay: true,
                             suggested_next_step: `datasets(action=preview, dataset_id=${datasetId})` });
                     }
-                    await putPresignedStream(init.upload_url, resolved, init.content_type ?? "application/octet-stream", PRESIGNED_PUT_TIMEOUT_MS);
+                    await putPresignedStream(init.upload_url, resolved, init.content_type ?? "application/octet-stream", PRESIGNED_PUT_TIMEOUT_MS, isGzipInput);
                     const fin = (await apiCall("POST", `/v1/datasets/${datasetId}/finalize`, {}));
                     const jobId = (fin.id ?? fin.job_id);
                     const poll = await pollUntilComplete(jobId, POLL_STAGE_MAX_MS);
@@ -267,6 +275,21 @@ ESCALATION: If upload fails with column errors, open the file locally and verify
                             : `Still staging; poll jobs(action=status, job_id="${jobId}") then datasets(action=preview, dataset_id=${datasetId}).`,
                     });
                 }
+                // Small pre-gzipped files: send the bytes as-is (the API decompresses
+                // Content-Encoding: gzip), never decoding them as UTF-8 text.
+                if (isGzipInput) {
+                    const gzBytes = await fs.readFile(resolved);
+                    const data = (await apiCall("POST", "/v1/datasets", gzBytes, {
+                        "X-Dataset-Name": name,
+                        "Content-Type": "text/csv",
+                        "Content-Encoding": "gzip",
+                        "Idempotency-Key": createHash("sha256").update(`${name}\n`).update(gzBytes).digest("hex"),
+                    }, UPLOAD_DATASET_TIMEOUT_MS));
+                    const gid = data.id ?? data.dataset_id;
+                    if (gid != null)
+                        data.suggested_next_step = `Suggested next step: datasets(action=preview, dataset_id=${gid}) to inspect columns before training.`;
+                    return textResult(data);
+                }
                 body = await fs.readFile(resolved, "utf-8");
             }
             else if (csv_data && csv_data.length > 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@barivia/barsom-mcp",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "barSOM MCP proxy — connect any MCP client to the barSOM cloud API for Self-Organizing Map analytics",
   "keywords": [
     "mcp",