@blamejs/core 0.8.12 → 0.8.15

package/lib/http-client.js

@@ -20,6 +20,10 @@
  *     maxResponseBytes,   // for buffer mode (default 16 MiB control,
  *                         //   1 GiB GET — operators with > 1 GiB
  *                         //   stored objects must use stream mode)
+ *     onChunk,            // (chunk: Buffer) => void — fires for each
+ *                         //   response chunk in BOTH buffer and stream
+ *                         //   modes. Use to hash bytes during pipe-to-disk
+ *                         //   without an extra Transform pass.
  *     signal,             // AbortSignal — propagated to req/stream
  *     errorClass,         // FrameworkError subclass
  *     observer,           // optional (stage, info) => void hook
@@ -372,6 +376,7 @@ function _toH2Headers(method, u, headers) {
   h2Headers[":path"]      = u.pathname + (u.search || "");
   h2Headers[":scheme"]    = u.protocol === "https:" ? "https" : "http";
   h2Headers[":authority"] = u.host;
+  var sawAcceptEncoding = false;
   for (var k in headers) {
     if (!Object.prototype.hasOwnProperty.call(headers, k)) continue;
     var lk = k.toLowerCase();
@@ -379,8 +384,12 @@ function _toH2Headers(method, u, headers) {
     if (lk === "connection" || lk === "host" ||
         lk === "keep-alive" || lk === "transfer-encoding" ||
         lk === "upgrade" || lk === "proxy-connection") continue;
+    if (lk === "accept-encoding") sawAcceptEncoding = true;
     h2Headers[lk] = headers[k];
   }
+  // CVE-2026-22036 mitigation — same identity default as the h1 path.
+  // Refuse compressed responses unless the operator explicitly opts in.
+  if (!sawAcceptEncoding) h2Headers["accept-encoding"] = "identity";
   return h2Headers;
 }
 
@@ -426,20 +435,78 @@ function _attachJarCookie(headers, jar, url) {
 // Mirrors the wire format that lib/middleware/body-parser.js's multipart
 // parser accepts so round-trip from one blamejs app's outbound to
 // another's inbound is exact.
+//
+// Two output shapes:
+//
+//   - { boundary, body: Buffer, contentLength }
+//       When every file entry is a Buffer / string (size known
+//       up front) and no operator opted into streaming, the result
+//       is a fully-materialized body. Smaller payloads avoid the
+//       streaming overhead and let HTTP/1.1 KeepAlive reuse with a
+//       known Content-Length.
+//
+//   - { boundary, body: Readable, contentLength }
+//       When at least one file entry is `{ filePath }` / `{ stream }`
+//       OR opts.streaming === true, the result is a Readable that
+//       emits boundary headers + content + CRLF in order. Avoids the
+//       Buffer.concat() OOM class on large uploads. contentLength is
+//       a finite number when every source's size is statically
+//       resolvable (Buffer length, fs.statSync().size, opts.size on
+//       a stream entry); null otherwise — caller falls back to
+//       chunked transfer.
+//
+// File entry shapes (all require `field`):
+//
+//   { field, content: Buffer | string }       — in-memory (existing)
+//   { field, filePath: string }               — stream-from-disk
+//   { field, stream: Readable, size?: number } — operator-supplied stream
+//
+// `filename` and `contentType` apply to all three shapes; for
+// `filePath` entries, `filename` defaults to path.basename(filePath).
 function _buildMultipartBody(spec) {
   var boundary = "----blamejs-mp-" + crypto.generateToken(C.BYTES.bytes(16));
   var CRLF = "\r\n";
-  var parts = [];
+  var fs = require("fs");                                             // allow:inline-require — only on multipart paths that touch the filesystem
+  var path = require("path");                                         // allow:inline-require — same
+  var nodeStream = require("stream");                                 // allow:inline-require — Readable subclass only when streaming
+
+  // Each entry is { headerBytes, source } where source is one of:
+  //   { kind: "buffer", buf: Buffer }
+  //   { kind: "filePath", filePath: string, size: number }
+  //   { kind: "stream", stream: Readable, size: number | null }
+  var entries = [];
+  var anyStreaming = false;
+  var totalSize = 0;
+  var sizeKnown = true;
+
+  function _entryHeaderBytes(disposition, contentType) {
+    var head = "--" + boundary + CRLF + disposition + CRLF;
+    if (contentType) head += "Content-Type: " + contentType + CRLF;
+    head += CRLF;
+    return Buffer.from(head, "utf8");
+  }
+
+  function _addEntry(headerBytes, source) {
+    entries.push({ header: headerBytes, source: source });
+    totalSize += headerBytes.length;
+    if (source.kind === "buffer") {
+      totalSize += source.buf.length;
+    } else if (typeof source.size === "number" && isFinite(source.size) && source.size >= 0) {
+      totalSize += source.size;
+    } else {
+      sizeKnown = false;
+    }
+    totalSize += CRLF.length;
+  }
 
   function _pushField(name, value) {
     if (typeof name !== "string" || name.length === 0) {
       throw new Error("multipart: field name must be a non-empty string");
     }
-    var head = "--" + boundary + CRLF +
-               'Content-Disposition: form-data; name="' + name + '"' + CRLF + CRLF;
-    parts.push(Buffer.from(head, "utf8"));
-    parts.push(Buffer.isBuffer(value) ? value : Buffer.from(String(value), "utf8"));
-    parts.push(Buffer.from(CRLF, "utf8"));
+    var disposition = 'Content-Disposition: form-data; name="' + name + '"';
+    var head = _entryHeaderBytes(disposition, null);
+    var bodyBuf = Buffer.isBuffer(value) ? value : Buffer.from(String(value), "utf8");
+    _addEntry(head, { kind: "buffer", buf: bodyBuf });
   }
 
   function _pushFile(file) {
@@ -447,21 +514,50 @@ function _buildMultipartBody(spec) {
     if (typeof file.field !== "string" || file.field.length === 0) {
       throw new Error("multipart: file.field must be a non-empty string");
     }
-    var filename = typeof file.filename === "string" && file.filename.length > 0
-      ? file.filename : "blob";
+    var hasContent  = file.content !== undefined && file.content !== null;
+    var hasFilePath = typeof file.filePath === "string" && file.filePath.length > 0;
+    var hasStream   = file.stream && typeof file.stream.pipe === "function";
+    var sourceCount = (hasContent ? 1 : 0) + (hasFilePath ? 1 : 0) + (hasStream ? 1 : 0);
+    if (sourceCount === 0) {
+      throw new Error("multipart: file entry requires one of { content, filePath, stream }");
+    }
+    if (sourceCount > 1) {
+      throw new Error("multipart: file entry must have exactly one of { content, filePath, stream }");
+    }
+
+    var filename;
+    if (typeof file.filename === "string" && file.filename.length > 0) {
+      filename = file.filename;
+    } else if (hasFilePath) {
+      filename = path.basename(file.filePath);
+    } else {
+      filename = "blob";
+    }
     var mimeType = file.contentType || file.mimeType || "application/octet-stream";
-    var content = file.content;
-    if (typeof content === "string") content = Buffer.from(content, "utf8");
-    if (!Buffer.isBuffer(content)) {
-      throw new Error("multipart: file.content must be a Buffer or string");
+    var disposition = 'Content-Disposition: form-data; name="' + file.field + '"' +
+                      '; filename="' + filename.replace(/"/g, "%22") + '"';
+    var head = _entryHeaderBytes(disposition, mimeType);
+
+    if (hasContent) {
+      var content = file.content;
+      if (typeof content === "string") content = Buffer.from(content, "utf8");
+      if (!Buffer.isBuffer(content)) {
+        throw new Error("multipart: file.content must be a Buffer or string");
+      }
+      _addEntry(head, { kind: "buffer", buf: content });
+    } else if (hasFilePath) {
+      anyStreaming = true;
+      var st;
+      try { st = fs.statSync(file.filePath); }
+      catch (e) { throw new Error("multipart: file.filePath not readable: " + e.message); }
+      if (!st.isFile()) throw new Error("multipart: file.filePath is not a regular file");
+      _addEntry(head, { kind: "filePath", filePath: file.filePath, size: st.size });
+    } else {
+      anyStreaming = true;
+      var streamSize = (typeof file.size === "number" && isFinite(file.size) && file.size >= 0)
+        ? file.size : null;
+      _addEntry(head, { kind: "stream", stream: file.stream, size: streamSize });
     }
-    var head = "--" + boundary + CRLF +
-               'Content-Disposition: form-data; name="' + file.field + '"' +
-                 '; filename="' + filename.replace(/"/g, "%22") + '"' + CRLF +
-               "Content-Type: " + mimeType + CRLF + CRLF;
-    parts.push(Buffer.from(head, "utf8"));
-    parts.push(content);
-    parts.push(Buffer.from(CRLF, "utf8"));
   }
 
   if (spec && spec.fields && typeof spec.fields === "object") {
@@ -479,8 +575,54 @@ function _buildMultipartBody(spec) {
   if (spec && Array.isArray(spec.files)) {
     for (var fi = 0; fi < spec.files.length; fi++) _pushFile(spec.files[fi]);
   }
-  parts.push(Buffer.from("--" + boundary + "--" + CRLF, "utf8"));
-  return { boundary: boundary, body: Buffer.concat(parts) };
+  var trailer = Buffer.from("--" + boundary + "--" + CRLF, "utf8");
+  totalSize += trailer.length;
+
+  // All-buffer fast path — return a fully-materialized body when no
+  // streaming sources are involved AND the operator didn't ask for
+  // streaming explicitly. Existing callers that pass small in-memory
+  // payloads keep the buffer codepath.
+  if (!anyStreaming && !(spec && spec.streaming === true)) {
+    var parts = [];
+    for (var ei = 0; ei < entries.length; ei++) {
+      parts.push(entries[ei].header);
+      parts.push(entries[ei].source.buf);
+      parts.push(Buffer.from(CRLF, "utf8"));
+    }
+    parts.push(trailer);
+    return { boundary: boundary, body: Buffer.concat(parts), contentLength: totalSize };
+  }
+
+  // Streaming path — produce a Readable from an async iterator that
+  // yields the bytes for each entry in order.
+  var crlfBuf = Buffer.from(CRLF, "utf8");
+  async function* _iter() {
+    for (var ix = 0; ix < entries.length; ix++) {
+      var entry = entries[ix];
+      yield entry.header;
+      if (entry.source.kind === "buffer") {
+        yield entry.source.buf;
+      } else if (entry.source.kind === "filePath") {
+        var rs = fs.createReadStream(entry.source.filePath);
+        try {
+          for await (var chunk of rs) yield chunk;
+        } finally {
+          try { rs.destroy(); } catch (_e) { /* best-effort cleanup */ }
+        }
+      } else {
+        // operator-supplied stream
+        for await (var chunk2 of entry.source.stream) yield chunk2;
+      }
+      yield crlfBuf;
+    }
+    yield trailer;
+  }
+  var body = nodeStream.Readable.from(_iter());
+  return {
+    boundary:      boundary,
+    body:          body,
+    contentLength: sizeKnown ? totalSize : null,
+  };
 }
 
 // Headers stripped on cross-origin redirect to defend against accidental
@@ -525,6 +667,10 @@ function request(opts) {
     return Promise.reject(_makeError(opts.errorClass, "BAD_ARG",
       "onDownloadProgress must be a function", true));
   }
+  if (opts.onChunk !== undefined && typeof opts.onChunk !== "function") {
+    return Promise.reject(_makeError(opts.errorClass, "BAD_ARG",
+      "onChunk must be a function (chunk: Buffer) -> void", true));
+  }
   if (opts.jar !== undefined && opts.jar !== null) {
     if (typeof opts.jar !== "object" ||
         typeof opts.jar.cookieHeaderFor !== "function" ||
@@ -567,13 +713,20 @@ function request(opts) {
     catch (e) {
       return Promise.reject(_makeError(opts.errorClass, "BAD_ARG", e.message, true));
     }
+    var mpHeaders = Object.assign({}, opts.headers || {}, {
+      "Content-Type": "multipart/form-data; boundary=" + built.boundary,
+    });
+    // Content-Length is set when the framework can statically resolve
+    // every source's byte size. Otherwise the framework omits the
+    // header and Node's HTTP layer falls back to chunked transfer —
+    // valid HTTP/1.1, requires no operator opt-in.
+    if (typeof built.contentLength === "number" && isFinite(built.contentLength)) {
+      mpHeaders["Content-Length"] = String(built.contentLength);
+    }
     opts = Object.assign({}, opts, {
-      method: opts.method || "POST",
-      body:   built.body,
-      headers: Object.assign({}, opts.headers || {}, {
-        "Content-Type":   "multipart/form-data; boundary=" + built.boundary,
-        "Content-Length": String(built.body.length),
-      }),
+      method:    opts.method || "POST",
+      body:      built.body,
+      headers:   mpHeaders,
       multipart: undefined,
     });
   }
@@ -872,6 +1025,17 @@ function _requestH1(transport, u, opts) {
     if (Buffer.isBuffer(opts.body)) {
       headers["Content-Length"] = opts.body.length;
     }
+    // CVE-2026-22036 mitigation — refuse compressed responses by
+    // default. The framework's http-client returns raw bytes capped
+    // at maxResponseBytes; if a server sends gzip/br/zstd the cap is
+    // on-wire bytes only, and any operator-side decompression is the
+    // operator's responsibility to bound. Identity by default closes
+    // the decompression-bomb amplification class. Operators who DO
+    // want compressed responses opt in by passing an explicit
+    // Accept-Encoding header (lowercase or canonical form).
+    if (!headers["Accept-Encoding"] && !headers["accept-encoding"]) {
+      headers["Accept-Encoding"] = "identity";
+    }
 
     var reqOpts = {
       method:   method,
@@ -894,6 +1058,7 @@ function _requestH1(transport, u, opts) {
 
     var onUploadProgress   = typeof opts.onUploadProgress === "function" ? opts.onUploadProgress : null;
     var onDownloadProgress = typeof opts.onDownloadProgress === "function" ? opts.onDownloadProgress : null;
+    var onChunk            = typeof opts.onChunk === "function" ? opts.onChunk : null;
 
     var req = transport.lib.request(reqOpts, function (res) {
       if (observer) observer("response:headers", { statusCode: res.statusCode, headers: res.headers });
@@ -927,13 +1092,24 @@ function _requestH1(transport, u, opts) {
             "HTTP " + res.statusCode + " " + (res.statusMessage || ""),
             _isPermanentStatus(res.statusCode), res.statusCode));
         }
-        if (onDownloadProgress) {
-          // Wrap the stream so chunks emit progress to the operator.
-          // The framework's contract is to hand back the response stream
-          // unmodified; fix-up via a passthrough keeps that contract while
-          // observing the chunk sizes.
+        if (onDownloadProgress || onChunk) {
+          // Wrap the stream so chunks emit progress + onChunk to the
+          // operator. The framework's contract is to hand back the
+          // response stream unmodified; fix-up via a passthrough keeps
+          // that contract while observing the chunk sizes. onChunk
+          // gets the buffer itself (for hash-as-you-go); a throw from
+          // it is caught and dropped so a hash-mismatch detector can
+          // raise without breaking the response stream — caller
+          // surfaces the error through their own pipe handler.
           var passthrough = new nodeStream.PassThrough();
-          res.on("data", function (chunk) { _emitDownload(chunk.length); passthrough.write(chunk); });
+          res.on("data", function (chunk) {
+            _emitDownload(chunk.length);
+            if (onChunk) {
+              try { onChunk(chunk); }
+              catch (_e) { /* operator-supplied hook — drop-silent */ }
+            }
+            passthrough.write(chunk);
+          });
           res.on("end",  function () { passthrough.end(); });
           res.on("error", function (e) { passthrough.destroy(e); });
           return _resolve({ statusCode: res.statusCode, headers: res.headers, body: passthrough });
@@ -955,6 +1131,10 @@ function _requestH1(transport, u, opts) {
           return;
         }
         _emitDownload(chunk.length);
+        if (onChunk) {
+          try { onChunk(chunk); }
+          catch (_e) { /* operator-supplied hook — drop-silent */ }
+        }
       });
       res.on("end", function () {
         if (capExceeded) return;
@@ -1072,6 +1252,7 @@ function _requestH2(transport, u, opts) {
       (method === "GET" ? DEFAULT_GET_CAP : DEFAULT_CONTROL_PLANE_CAP);
     var observer = typeof opts.observer === "function" ? opts.observer : null;
     var startedAt = Date.now();
+    var onChunkH2 = typeof opts.onChunk === "function" ? opts.onChunk : null;
 
     var signal = safeAsync.withTimeoutSignal(opts.signal || null, opts.timeoutMs);
     if (signal && signal.aborted) {
@@ -1128,6 +1309,17 @@ function _requestH2(transport, u, opts) {
           return _reject(_makeError(opts.errorClass, "HTTP_ERROR",
             "HTTP " + statusCode, _isPermanentStatus(statusCode), statusCode));
         }
+        if (onChunkH2) {
+          var passthroughH2 = new nodeStream.PassThrough();
+          stream.on("data", function (chunk) {
+            try { onChunkH2(chunk); }
+            catch (_e) { /* operator-supplied hook — drop-silent */ }
+            passthroughH2.write(chunk);
+          });
+          stream.on("end",  function () { passthroughH2.end(); });
+          stream.on("error", function (e) { passthroughH2.destroy(e); });
+          return _resolve({ statusCode: statusCode, headers: responseHeaders, body: passthroughH2 });
+        }
         return _resolve({ statusCode: statusCode, headers: responseHeaders, body: stream });
       }
 
@@ -1142,6 +1334,11 @@ function _requestH2(transport, u, opts) {
           try { stream.close(http2.constants.NGHTTP2_CANCEL); } catch (_e2) { /* best-effort h2 stream cancel */ }
           _reject(_makeError(opts.errorClass, "RESPONSE_TOO_LARGE",
             "response body exceeds " + maxResponseBytes + " bytes", true));
+          return;
+        }
+        if (onChunkH2) {
+          try { onChunkH2(chunk); }
+          catch (_e) { /* operator-supplied hook — drop-silent */ }
         }
       });
       stream.on("end", function () {