@codemation/core-nodes 0.6.0 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codemation/core-nodes",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "publishConfig": {
     "access": "public"
   },
@@ -33,7 +33,7 @@
     "ai": "^6.0.168",
     "croner": "^10.0.1",
     "lucide-react": "^0.577.0",
-    "@codemation/core": "0.10.0"
+    "@codemation/core": "0.10.1"
   },
   "devDependencies": {
     "@types/node": "^25.3.5",

package/src/http/HttpBodyBuilder.ts

@@ -1,3 +1,5 @@
+import type { ReadableStream as NodeReadableStream } from "node:stream/web";
+
 import type { Item, NodeExecutionContext } from "@codemation/core";
 import type { RunnableNodeConfig } from "@codemation/core";
 import type { HttpBodySpec } from "./httpRequest.types";
@@ -54,23 +56,7 @@ export class HttpBodyBuilder {
           if (attachment) {
             const readResult = await ctx.binary.openReadStream(attachment);
             if (readResult) {
-              const reader = readResult.body.getReader();
-              const chunks: Uint8Array[] = [];
-              let done = false;
-              while (!done) {
-                const result = await reader.read();
-                done = result.done;
-                if (result.value) {
-                  chunks.push(result.value);
-                }
-              }
-              const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
-              const merged = new Uint8Array(totalLength);
-              let offset = 0;
-              for (const chunk of chunks) {
-                merged.set(chunk, offset);
-                offset += chunk.length;
-              }
+              const merged = await this.readStreamToBuffer(readResult.body);
               const blob = new Blob([merged], { type: attachment.mimeType });
               formData.append(fieldName, blob, attachment.filename ?? binaryRef);
             }
@@ -85,6 +71,48 @@ export class HttpBodyBuilder {
       };
     }
 
+    if (spec.kind === "binary") {
+      const attachment = item.binary?.[spec.slot];
+      if (!attachment) {
+        throw new Error(
+          `HttpRequest bodyFormat "binary": no binary attachment found at slot "${spec.slot}". ` +
+            `Ensure a previous node attached binary data at that slot.`,
+        );
+      }
+      const readResult = await ctx.binary.openReadStream(attachment);
+      if (!readResult) {
+        throw new Error(`HttpRequest bodyFormat "binary": could not open read stream for slot "${spec.slot}".`);
+      }
+      // Pass the stream straight to fetch — no buffering. fetch's BodyInit
+      // accepts a ReadableStream natively, so big attachments upload without
+      // ever materialising the full payload in memory.
+      return {
+        body: readResult.body as unknown as NonNullable<RequestInit["body"]>,
+        contentType: attachment.mimeType,
+      };
+    }
+
     return undefined;
   }
+
+  private async readStreamToBuffer(stream: NodeReadableStream<Uint8Array>): Promise<Uint8Array<ArrayBuffer>> {
+    const reader = stream.getReader();
+    const chunks: Uint8Array[] = [];
+    let done = false;
+    while (!done) {
+      const result = await reader.read();
+      done = result.done;
+      if (result.value) {
+        chunks.push(result.value);
+      }
+    }
+    const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
+    const merged = new Uint8Array(new ArrayBuffer(totalLength));
+    let offset = 0;
+    for (const chunk of chunks) {
+      merged.set(chunk, offset);
+      offset += chunk.length;
+    }
+    return merged;
+  }
 }

package/src/http/HttpRequestExecutor.ts

@@ -46,7 +46,10 @@ export class HttpRequestExecutor {
 
     // Only set Content-Type from the encoded body when it is non-empty
     // (empty string = FormData will set it automatically).
-    if (encodedBody && encodedBody.contentType) {
+    // Explicit headers always win — only apply the body-derived content-type
+    // when the caller has not already set one (case-insensitive check).
+    const hasExplicitContentType = Object.keys(mergedHeaders).some((k) => k.toLowerCase() === "content-type");
+    if (encodedBody && encodedBody.contentType && !hasExplicitContentType) {
       mergedHeaders["content-type"] = encodedBody.contentType;
     }
 

package/src/http/httpRequest.types.ts

@@ -17,6 +17,16 @@ export type HttpBodySpec =
       kind: "multipart";
       fields: Readonly<Record<string, string>>;
       binaries?: Readonly<Record<string, BinaryRef>>;
+    }>
+  | Readonly<{
+      /**
+       * Send raw bytes from a binary slot as the request body.
+       * The binary attachment's `mimeType` is used as `Content-Type` unless
+       * the request `headers` map already contains `content-type`.
+       */
+      kind: "binary";
+      /** Key into `item.binary` to read the request body bytes from. */
+      slot: string;
     }>;
 
 /**
@@ -48,6 +58,15 @@ export type HttpRequestSpec = Readonly<{
   body?: HttpBodySpec;
   credential?: CredentialSession;
   download?: Readonly<{ mode: "auto" | "always" | "never"; binaryName: string }>;
+  /**
+   * When set to `"binary"`, the response body is written to a binary slot
+   * instead of being parsed as JSON/text. Overrides `download` mode.
+   */
+  responseFormat?: "json" | "text" | "binary";
+  /** Binary slot name for the response body when `responseFormat === "binary"`. Defaults to `"response"`. */
+  responseBinarySlot?: string;
+  /** Maximum allowed response size in bytes (checked against Content-Length before allocating). Defaults to 100 MiB. */
+  responseSizeCapBytes?: number;
   /** Execution context — needed for binary attach. */
   ctx: NodeExecutionContext<RunnableNodeConfig<unknown, unknown>>;
 }>;
@@ -66,4 +85,12 @@ export type HttpRequestResult = Readonly<{
   json?: unknown;
   text?: string;
   bodyBinaryName?: string;
+  /** Set when `responseFormat === "binary"`. Name of the binary slot the response body was written to. */
+  binarySlot?: string;
+  /** Set when `responseFormat === "binary"`. The MIME type of the stored response. */
+  contentType?: string;
+  /** Set when `responseFormat === "binary"`. Size in bytes of the stored response. */
+  size?: number;
+  /** Set when `responseFormat === "binary"`. Filename inferred from URL or Content-Disposition. */
+  filename?: string;
 }>;

package/src/nodes/HttpRequestNodeFactory.ts

@@ -32,6 +32,9 @@ export class HttpRequestNode implements RunnableNode<HttpRequest<any, any>> {
         mode: ctx.config.downloadMode,
         binaryName: ctx.config.binaryName,
       },
+      responseFormat: ctx.config.responseFormat,
+      responseBinarySlot: ctx.config.responseBinarySlot,
+      responseSizeCapBytes: ctx.config.responseSizeCapBytes,
       ctx: ctx as unknown as HttpRequestSpec["ctx"],
     };
 
@@ -45,6 +48,12 @@ export class HttpRequestNode implements RunnableNode<HttpRequest<any, any>> {
 
     const headers = this.readHeaders(response.headers);
     const mimeType = this.resolveMimeType(headers);
+
+    // New explicit responseFormat="binary" path — takes precedence over downloadMode.
+    if (ctx.config.responseFormat === "binary") {
+      return await this.handleBinaryResponse(response, resolvedUrl, headers, mimeType, ctx);
+    }
+
     const binaryName = ctx.config.binaryName;
     const shouldAttach = this.shouldAttachBody(ctx.config.downloadMode, mimeType);
 
@@ -64,7 +73,8 @@ export class HttpRequestNode implements RunnableNode<HttpRequest<any, any>> {
         name: binaryName,
         body: response.body
           ? (response.body as unknown as Parameters<typeof ctx.binary.attach>[0]["body"])
-          : new Uint8Array(await response.arrayBuffer()),
+          : // eslint-disable-next-line codemation/no-buffer-everything -- response.body is null (e.g. 204/304); fallback path is intentional and bounded.
+            new Uint8Array(await response.arrayBuffer()),
         mimeType,
         filename: this.resolveFilename(resolvedUrl, headers),
       });
@@ -104,6 +114,64 @@ export class HttpRequestNode implements RunnableNode<HttpRequest<any, any>> {
     return { json: outputJson };
   }
 
+  private async handleBinaryResponse(
+    response: Response,
+    resolvedUrl: string,
+    headers: Readonly<Record<string, string>>,
+    mimeType: string,
+    ctx: NodeExecutionContext<HttpRequest<any, any>>,
+  ): Promise<Item> {
+    const slotName = ctx.config.responseBinarySlot;
+    const sizeCap = ctx.config.responseSizeCapBytes;
+
+    // Check Content-Length against size cap before allocating.
+    const contentLengthHeader = headers["content-length"];
+    if (contentLengthHeader) {
+      const declaredSize = parseInt(contentLengthHeader, 10);
+      if (!isNaN(declaredSize) && declaredSize > sizeCap) {
+        throw new Error(
+          `HttpRequest responseFormat "binary": response Content-Length (${declaredSize} bytes) ` +
+            `exceeds responseSizeCapBytes (${sizeCap} bytes).`,
+        );
+      }
+    }
+
+    const filename = this.resolveFilename(resolvedUrl, headers);
+
+    // Stream response.body straight into binary storage — never load the
+    // whole payload into memory. ctx.binary.attach accepts ReadableStream
+    // natively. Falls back to arrayBuffer only when response.body is null
+    // (rare; 204/304-style responses where the cap-check above already
+    // covers the meaningful size case).
+    const attachment = await ctx.binary.attach({
+      name: slotName,
+      body: response.body
+        ? (response.body as unknown as Parameters<typeof ctx.binary.attach>[0]["body"])
+        : // eslint-disable-next-line codemation/no-buffer-everything -- response.body is null on 204/304-style empty responses; the size-cap check above already gates large bodies, so buffering an empty payload here is bounded and unavoidable.
+          new Uint8Array(await response.arrayBuffer()),
+      mimeType,
+      filename,
+    });
+
+    const outputJson: Readonly<Record<string, unknown>> = {
+      url: resolvedUrl,
+      method: ctx.config.method,
+      ok: response.ok,
+      status: response.status,
+      statusText: response.statusText,
+      headers,
+      binarySlot: slotName,
+      contentType: mimeType,
+      // Reported by the binary storage adapter after streaming completes.
+      size: attachment.size,
+      ...(filename !== undefined ? { filename } : {}),
+    };
+
+    let outputItem: Item = { json: outputJson };
+    outputItem = ctx.binary.withAttachment(outputItem, slotName, attachment);
+    return outputItem;
+  }
+
   private async resolveCredential(
     ctx: NodeExecutionContext<HttpRequest<any, any>>,
   ): Promise<CredentialSession | undefined> {

package/src/nodes/httpRequest.ts

@@ -29,6 +29,14 @@ export type HttpRequestOutputJson = Readonly<{
   json?: unknown;
   text?: string;
   bodyBinaryName?: string;
+  /** Set when `responseFormat === "binary"`. Name of the binary slot the response was stored in. */
+  binarySlot?: string;
+  /** Set when `responseFormat === "binary"`. MIME type of the stored response. */
+  contentType?: string;
+  /** Set when `responseFormat === "binary"`. Size in bytes of the stored response. */
+  size?: number;
+  /** Set when `responseFormat === "binary"`. Filename inferred from URL or Content-Disposition. */
+  filename?: string;
 }>;
 
 /**
@@ -42,6 +50,9 @@ export const HTTP_REQUEST_ACCEPTED_CREDENTIAL_TYPES: ReadonlyArray<string> = [
   oauth2ClientCredentialsType.definition.typeId,
 ] as const;
 
+/** Default maximum response size for binary mode: 100 MiB. */
+const DEFAULT_RESPONSE_SIZE_CAP_BYTES = 100 * 1024 * 1024;
+
 export class HttpRequest<
   TInputJson = Readonly<{ url?: string }>,
   TOutputJson = HttpRequestOutputJson,
@@ -77,6 +88,27 @@ export class HttpRequest<
       credentialSlot?: string;
       binaryName?: string;
       downloadMode?: HttpRequestDownloadMode;
+      /**
+       * Controls how the response body is handled.
+       * - `"json"` / `"text"`: existing behaviour (parse + emit on `item.json`).
+       * - `"binary"`: read the response as raw bytes and store via `ctx.binary.attach`.
+       *   The output JSON contains `{ status, headers, binarySlot, contentType, size, filename }`
+       *   but NOT the raw bytes. Use `responseBinarySlot` to name the slot (default `"response"`).
+       *
+       * When omitted, the existing `downloadMode` logic applies (backward-compatible).
+       */
+      responseFormat?: "json" | "text" | "binary";
+      /**
+       * Name of the binary slot to write the response body into when `responseFormat === "binary"`.
+       * Defaults to `"response"`.
+       */
+      responseBinarySlot?: string;
+      /**
+       * Maximum response size in bytes for binary mode. Checked against the `Content-Length`
+       * response header before allocating memory. Defaults to 100 MiB (104857600).
+       * Requests whose `Content-Length` exceeds this cap are rejected before the body is read.
+       */
+      responseSizeCapBytes?: number;
       id?: string;
     }> = {},
     public readonly retryPolicy: RetryPolicySpec = RetryPolicy.defaultForHttp,
@@ -102,6 +134,18 @@ export class HttpRequest<
     return this.args.downloadMode ?? "auto";
   }
 
+  get responseFormat(): "json" | "text" | "binary" | undefined {
+    return this.args.responseFormat;
+  }
+
+  get responseBinarySlot(): string {
+    return this.args.responseBinarySlot ?? "response";
+  }
+
+  get responseSizeCapBytes(): number {
+    return this.args.responseSizeCapBytes ?? DEFAULT_RESPONSE_SIZE_CAP_BYTES;
+  }
+
   getCredentialRequirements(): ReadonlyArray<CredentialRequirement> {
     if (!this.args.credentialSlot) {
       return [];