@opennextjs/aws 3.8.0 → 3.8.2

package/README.md

@@ -98,7 +98,28 @@ To run `OpenNext` locally:
 
 There is also a way to run OpenNext locally. You can read a guide about it [here](https://opennext.js.org/aws/contribute/local_run). Its mostly used for development/debugging purposes.
 
-### Coldstart
+## Testing
+
+You can run unit tests with
+
+```bash
+pnpm test
+```
+
+You can tun e2e locally with:
+
+```bash
+pnpm -r openbuild:local
+pnpm -r openbuild:local:start
+```
+
+And in a different shell:
+
+```bash
+pnpm e2e:test
+```
+
+## Coldstart
 
 OpenNext provide you with a warmer function that can be used to reduce cold start.
 

package/dist/build/createImageOptimizationBundle.js

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import { createRequire } from "node:module";
+import os from "node:os";
 import path from "node:path";
 import logger from "../logger.js";
 import { openNextReplacementPlugin } from "../plugins/replacement.js";
@@ -74,10 +75,14 @@ export async function createImageOptimizationBundle(options) {
     // Target should be same as used by Lambda, see https://github.com/sst/sst/blob/ca6f763fdfddd099ce2260202d0ce48c72e211ea/packages/sst/src/constructs/NextjsSite.ts#L114
     // For SHARP_IGNORE_GLOBAL_LIBVIPS see: https://github.com/lovell/sharp/blob/main/docs/install.md#aws-lambda
     const sharpVersion = process.env.SHARP_VERSION ?? "0.32.6";
+    // In development, we want to use the local machine settings
+    const isDev = config.imageOptimization?.loader === "fs-dev";
     installDependencies(outputPath, config.imageOptimization?.install ?? {
         packages: [`sharp@${sharpVersion}`],
-        // By not specifying an arch, `npm install` will choose one for us (i.e. our system one)
-        arch: config.imageOptimization?.loader === "fs-dev" ? undefined : "arm64",
+        // By not specifying an arch in dev, `npm install` will choose one for us (i.e. our system one)
+        arch: isDev ? undefined : "arm64",
+        // Use the local platform in dev
+        os: isDev ? os.platform() : "linux",
         nodeVersion: "18",
         libc: "glibc",
     });

package/dist/build/installDeps.js

@@ -24,8 +24,9 @@ export function installDependencies(outputDir, installOptions) {
         const libcOption = installOptions.libc
             ? `--libc=${installOptions.libc}`
             : "";
+        const osOption = `--os=${installOptions.os ?? "linux"}`;
         const additionalArgs = installOptions.additionalArgs ?? "";
-        const installCommand = `npm install --platform=linux ${archOption} ${targetOption} ${libcOption} ${additionalArgs} ${installOptions.packages.join(" ")}`;
+        const installCommand = `npm install ${osOption} ${archOption} ${targetOption} ${libcOption} ${additionalArgs} ${installOptions.packages.join(" ")}`;
         execSync(installCommand, {
             stdio: "pipe",
             cwd: tempInstallDir,

package/dist/core/routing/util.d.ts

@@ -3,6 +3,7 @@ import type { IncomingMessage } from "../../http/index.js";
 import { OpenNextNodeResponse } from "../../http/openNextResponse.js";
 import type { FunctionsConfigManifest, MiddlewareManifest } from "../../types/next-types";
 import type { InternalEvent, InternalResult, RoutingResult, StreamCreator } from "../../types/open-next.js";
+import { ReadableStream } from "node:stream/web";
 /**
  *
  * @__PURE__

package/dist/core/routing/util.js

@@ -1,9 +1,9 @@
 import crypto from "node:crypto";
 import { parse as parseQs, stringify as stringifyQs } from "node:querystring";
-import { Readable } from "node:stream";
 import { BuildId, HtmlPages, NextConfig } from "../../adapters/config/index.js";
 import { OpenNextNodeResponse } from "../../http/openNextResponse.js";
 import { getQueryFromIterator, parseHeaders } from "../../http/util.js";
+import { ReadableStream } from "node:stream/web";
 import { debug, error } from "../../adapters/logger.js";
 import { isBinaryContentType } from "../../utils/binary.js";
 import { localizePath } from "./i18n/index.js";
@@ -84,9 +84,15 @@ export function convertRes(res) {
     const headers = parseHeaders(res.getFixedHeaders());
     const isBase64Encoded = isBinaryContentType(headers["content-type"]) ||
         !!headers["content-encoding"];
-    // We cannot convert the OpenNextNodeResponse to a ReadableStream directly
-    // You can look in the `aws-lambda.ts` file for some context
-    const body = Readable.toWeb(Readable.from(res.getBody()));
+    const body = new ReadableStream({
+        pull(controller) {
+            if (!res._chunks || res._chunks.length === 0) {
+                controller.close();
+                return;
+            }
+            controller.enqueue(res._chunks.shift());
+        },
+    });
     return {
         type: "core",
         statusCode,
@@ -171,13 +177,12 @@ export function convertBodyToReadableStream(method, body) {
         return undefined;
     if (!body)
         return undefined;
-    const readable = new ReadableStream({
+    return new ReadableStream({
         start(controller) {
             controller.enqueue(body);
             controller.close();
         },
     });
-    return readable;
 }
 var CommonHeaders;
 (function (CommonHeaders) {

package/dist/http/openNextResponse.d.ts

@@ -11,10 +11,11 @@ export declare class OpenNextNodeResponse extends Transform implements ServerRes
     statusCode: number;
     statusMessage: string;
     headers: OutgoingHttpHeaders;
-    private _cookies;
-    private responseStream?;
     headersSent: boolean;
     _chunks: Buffer[];
+    private _cookies;
+    private responseStream?;
+    private bodyLength;
     strictContentLength: boolean;
     assignSocket(_socket: Socket): void;
     detachSocket(_socket: Socket): void;

package/dist/http/openNextResponse.js

@@ -12,10 +12,11 @@ export class OpenNextNodeResponse extends Transform {
     statusCode;
     statusMessage = "";
     headers = {};
-    _cookies = [];
-    responseStream;
     headersSent = false;
     _chunks = [];
+    _cookies = [];
+    responseStream;
+    bodyLength = 0;
     // To comply with the ServerResponse interface :
     strictContentLength = false;
     assignSocket(_socket) {
@@ -221,7 +222,12 @@ export class OpenNextNodeResponse extends Transform {
         return Buffer.concat(this._chunks);
     }
     _internalWrite(chunk, encoding) {
-        this._chunks.push(Buffer.from(chunk, encoding));
+        const buffer = Buffer.from(chunk, encoding);
+        this.bodyLength += buffer.length;
+        if (this.streamCreator?.retainChunks !== false) {
+            // Avoid keeping chunks around when the `StreamCreator` supports it to save memory
+            this._chunks.push(buffer);
+        }
         this.push(chunk, encoding);
         this.streamCreator?.onWrite?.();
     }
@@ -237,19 +243,18 @@ export class OpenNextNodeResponse extends Transform {
             this.flushHeaders();
         }
         // In some cases we might not have a store i.e. for example in the image optimization function
-        // We may want to reconsider this in the future, it might be intersting to have access to this store everywhere
+        // We may want to reconsider this in the future, it might be interesting to have access to this store everywhere
         globalThis.__openNextAls
             ?.getStore()
             ?.pendingPromiseRunner.add(this.onEnd(this.headers));
-        const bodyLength = this.getBody().length;
-        this.streamCreator?.onFinish?.(bodyLength);
+        this.streamCreator?.onFinish?.(this.bodyLength);
         //This is only here because of aws broken streaming implementation.
         //Hopefully one day they will be able to give us a working streaming implementation in lambda for everyone
         //If you're lucky you have a working streaming implementation in your aws account and don't need this
         //If not you can set the OPEN_NEXT_FORCE_NON_EMPTY_RESPONSE env variable to true
         //BE CAREFUL: Aws keeps rolling out broken streaming implementations even on accounts that had working ones before
         //This is not dependent on the node runtime used
-        if (bodyLength === 0 &&
+        if (this.bodyLength === 0 &&
             // We use an env variable here because not all aws account have the same behavior
             // On some aws accounts the response will hang if the body is empty
             // We are modifying the response body here, this is not a good practice
@@ -283,8 +288,10 @@ export class OpenNextNodeResponse extends Transform {
         return (Array.isArray(values) ? values : [values]).map((value) => value.toString());
     }
     send() {
-        const body = this.getBody();
-        this.end(body);
+        for (const chunk of this._chunks) {
+            this.write(chunk);
+        }
+        this.end();
     }
     body(value) {
         this.write(value);

package/dist/http/util.js

@@ -1,3 +1,4 @@
+import logger from "../logger";
 export const parseHeaders = (headers) => {
     const result = {};
     if (!headers) {
@@ -7,7 +8,26 @@ export const parseHeaders = (headers) => {
         if (value === undefined) {
             continue;
         }
-        result[key.toLowerCase()] = convertHeader(value);
+        const keyLower = key.toLowerCase();
+        /**
+         * Next can return an Array for the Location header when you return null from a get in the cacheHandler on a page that has a redirect()
+         * We dont want to merge that into a comma-separated string
+         * If they are the same just return one of them
+         * Otherwise return the last one
+         * See: https://github.com/opennextjs/opennextjs-cloudflare/issues/875#issuecomment-3258248276
+         * and https://github.com/opennextjs/opennextjs-aws/pull/977#issuecomment-3261763114
+         */
+        if (keyLower === "location" && Array.isArray(value)) {
+            if (value[0] === value[1]) {
+                result[keyLower] = value[0];
+            }
+            else {
+                logger.warn("Multiple different values for Location header found. Using the last one");
+                result[keyLower] = value[value.length - 1];
+            }
+            continue;
+        }
+        result[keyLower] = convertHeader(value);
     }
     return result;
 };

package/dist/overrides/wrappers/cloudflare-node.js

@@ -42,6 +42,8 @@ const handler = async (handler, converter) => async (request, env, ctx, abortSig
         // Ensures that the response we pass to NextServer is aborted if the request is aborted
         // By doing this `request.signal.onabort` will work in route handlers
         abortSignal: abortSignal,
+        // There is no need to retain the chunks that were pushed to the response stream.
+        retainChunks: false,
     };
     ctx.waitUntil(handler(internalEvent, {
         streamCreator,

package/dist/types/open-next.d.ts

@@ -36,6 +36,18 @@ export interface StreamCreator {
     onWrite?: () => void;
     onFinish?: (length: number) => void;
     abortSignal?: AbortSignal;
+    /**
+     * Normally there is no need to retain the chunks that have been pushed to the response stream.
+     *
+     * However some implementations use a fake `StreamCreator` and expect the chunks to be retained.
+     * When your stream controller implementation doesn't need to retain the chunk, you can set this
+     * to `false` to reduce memory usage.
+     *
+     * @see https://github.com/opennextjs/opennextjs-aws/blob/main/packages/open-next/src/overrides/wrappers/aws-lambda.ts
+     *
+     * @default true for backward compatibility.
+     */
+    retainChunks?: boolean;
 }
 export type WaitUntil = (promise: Promise<void>) => void;
 export interface DangerousOptions {
@@ -178,18 +190,14 @@ export interface InstallOptions {
      * ```
      */
     packages: string[];
-    /**
-     * @default undefined
-     */
+    /** @default undefined */
     arch?: "x64" | "arm64";
-    /**
-     * @default undefined
-     */
+    /** @default undefined */
     nodeVersion?: string;
-    /**
-     * @default undefined
-     */
+    /** @default undefined */
     libc?: "glibc" | "musl";
+    /** @default "linux" */
+    os?: string;
     /**
      * @default undefined
      * Additional arguments to pass to the install command (i.e. npm install)

package/dist/utils/binary.js

@@ -61,6 +61,6 @@ const commonBinaryMimeTypes = new Set([
 export function isBinaryContentType(contentType) {
     if (!contentType)
         return false;
-    const value = contentType?.split(";")[0] ?? "";
+    const value = contentType.split(";")[0];
     return commonBinaryMimeTypes.has(value);
 }

package/dist/utils/fetch.js

@@ -17,7 +17,6 @@ export function customFetchClient(client) {
          * This is necessary otherwise we get some error : SocketError: other side closed
          * https://github.com/nodejs/undici/issues/583#issuecomment-855384858
          */
-        const clonedResponse = response.clone();
-        return clonedResponse;
+        return response.clone();
     };
 }

package/dist/utils/stream.d.ts

@@ -1,4 +1,4 @@
-import type { ReadableStream } from "node:stream/web";
+import { ReadableStream } from "node:stream/web";
 export declare function fromReadableStream(stream: ReadableStream<Uint8Array>, base64?: boolean): Promise<string>;
 export declare function toReadableStream(value: string, isBase64?: boolean): ReadableStream;
 export declare function emptyReadableStream(): ReadableStream;

package/dist/utils/stream.js

@@ -1,30 +1,49 @@
-import { Readable } from "node:stream";
-export function fromReadableStream(stream, base64) {
-    const reader = stream.getReader();
+import { ReadableStream } from "node:stream/web";
+export async function fromReadableStream(stream, base64) {
     const chunks = [];
-    return new Promise((resolve, reject) => {
-        function pump() {
-            reader
-                .read()
-                .then(({ done, value }) => {
-                if (done) {
-                    resolve(Buffer.concat(chunks).toString(base64 ? "base64" : "utf8"));
-                    return;
-                }
-                chunks.push(value);
-                pump();
-            })
-                .catch(reject);
-        }
-        pump();
-    });
+    let totalLength = 0;
+    for await (const chunk of stream) {
+        chunks.push(chunk);
+        totalLength += chunk.length;
+    }
+    if (chunks.length === 0) {
+        return "";
+    }
+    if (chunks.length === 1) {
+        return Buffer.from(chunks[0]).toString(base64 ? "base64" : "utf8");
+    }
+    // Pre-allocate buffer with exact size to avoid reallocation
+    const buffer = Buffer.alloc(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        buffer.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return buffer.toString(base64 ? "base64" : "utf8");
 }
 export function toReadableStream(value, isBase64) {
-    return Readable.toWeb(Readable.from(Buffer.from(value, isBase64 ? "base64" : "utf8")));
+    return new ReadableStream({
+        pull(controller) {
+            // Defer the Buffer.from conversion to when the stream is actually read.
+            controller.enqueue(Buffer.from(value, isBase64 ? "base64" : "utf8"));
+            controller.close();
+        },
+    }, { highWaterMark: 0 });
 }
+let maybeSomethingBuffer;
 export function emptyReadableStream() {
     if (process.env.OPEN_NEXT_FORCE_NON_EMPTY_RESPONSE === "true") {
-        return Readable.toWeb(Readable.from([Buffer.from("SOMETHING")]));
+        return new ReadableStream({
+            pull(controller) {
+                maybeSomethingBuffer ??= Buffer.from("SOMETHING");
+                controller.enqueue(maybeSomethingBuffer);
+                controller.close();
+            },
+        }, { highWaterMark: 0 });
     }
-    return Readable.toWeb(Readable.from([]));
+    return new ReadableStream({
+        start(controller) {
+            controller.close();
+        },
+    });
 }

package/package.json

@@ -3,7 +3,7 @@
     "access": "public"
   },
   "name": "@opennextjs/aws",
-  "version": "3.8.0",
+  "version": "3.8.2",
   "bin": {
     "open-next": "./dist/index.js"
   },