@opennextjs/cloudflare 1.10.0 → 1.11.0

package/README.md

@@ -55,3 +55,30 @@ Deploy your application to production with the following:
   # or
   bun opennextjs-cloudflare build && bun opennextjs-cloudflare deploy
   ```
+
+### Batch Cache Population (Optional, Recommended)
+
+For improved performance with large caches, you can enable batch upload by providing R2 credentials via .env or environment variables.
+
+Create a `.env` file in your project root (automatically loaded by the CLI):
+
+```bash
+R2_ACCESS_KEY_ID=your_access_key_id
+R2_SECRET_ACCESS_KEY=your_secret_access_key
+CF_ACCOUNT_ID=your_account_id
+```
+
+You can also set the environment variables for CI builds.
+
+**Note:**
+
+You can follow documentation https://developers.cloudflare.com/r2/api/tokens/ for creating API tokens with appropriate permissions for R2 access.
+
+**Benefits:**
+
+- Significantly faster uploads for large caches using parallel transfers
+- Reduced API calls to Cloudflare
+- Automatically enabled when credentials are provided
+
+**Fallback:**
+If these environment variables are not set, the CLI will use standard Wrangler uploads. Both methods work correctly - batch upload is simply faster for large caches.

package/dist/api/cloudflare-context.d.ts

@@ -31,6 +31,8 @@ declare global {
         CF_PREVIEW_DOMAIN?: string;
         CF_WORKERS_SCRIPTS_API_TOKEN?: string;
         CF_ACCOUNT_ID?: string;
+        R2_ACCESS_KEY_ID?: string;
+        R2_SECRET_ACCESS_KEY?: string;
     }
 }
 export type CloudflareContext<CfProperties extends Record<string, unknown> = IncomingRequestCfProperties, Context = ExecutionContext> = {

package/dist/cli/build/build.js

@@ -13,7 +13,6 @@ import { compileInit } from "./open-next/compile-init.js";
 import { compileSkewProtection } from "./open-next/compile-skew-protection.js";
 import { compileDurableObjects } from "./open-next/compileDurableObjects.js";
 import { createServerBundle } from "./open-next/createServerBundle.js";
-import { createWranglerConfigIfNotExistent } from "./utils/index.js";
 import { getVersion } from "./utils/version.js";
 /**
  * Builds the application in a format that can be passed to workerd
@@ -65,9 +64,6 @@ export async function build(options, config, projectOpts, wranglerConfig) {
     await createServerBundle(options);
     await compileDurableObjects(options);
     await bundleServer(options, projectOpts);
-    if (!projectOpts.skipWranglerConfigCheck) {
-        await createWranglerConfigIfNotExistent(projectOpts);
-    }
     logger.info("OpenNext build complete.");
 }
 function ensureNextjsVersionSupported(options) {

package/dist/cli/commands/build.js

@@ -1,4 +1,5 @@
 import { build as buildImpl } from "../build/build.js";
+import { createWranglerConfigIfNotExistent } from "../build/utils/index.js";
 import { compileConfig, getNormalizedOptions, nextAppDir, printHeaders, readWranglerConfig, withWranglerOptions, withWranglerPassthroughArgs, } from "./utils.js";
 /**
  * Implementation of the `opennextjs-cloudflare build` command.
@@ -9,8 +10,15 @@ async function buildCommand(args) {
     printHeaders("build");
     const { config, buildDir } = await compileConfig(args.openNextConfigPath);
     const options = getNormalizedOptions(config, buildDir);
+    const projectOpts = { ...args, minify: !args.noMinify, sourceDir: nextAppDir };
+    // Ask whether a `wrangler.jsonc` should be created when no config file exists.
+    // Note: We don't ask when a custom config file is specified via `--config`
+    //       nor when `--skipWranglerConfigCheck` is used.
+    if (!projectOpts.wranglerConfigPath && !args.skipWranglerConfigCheck) {
+        await createWranglerConfigIfNotExistent(projectOpts);
+    }
     const wranglerConfig = readWranglerConfig(args);
-    await buildImpl(options, config, { ...args, minify: !args.noMinify, sourceDir: nextAppDir }, wranglerConfig);
+    await buildImpl(options, config, projectOpts, wranglerConfig);
 }
 /**
  * Add the `build` command to yargs configuration.

package/dist/cli/commands/populate-cache.js

@@ -1,7 +1,9 @@
-import { cpSync, existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
 import path from "node:path";
 import logger from "@opennextjs/aws/logger.js";
 import { globSync } from "glob";
+import rclone from "rclone.js";
 import { tqdm } from "ts-tqdm";
 import { BINDING_NAME as KV_CACHE_BINDING_NAME, NAME as KV_CACHE_NAME, PREFIX_ENV_NAME as KV_CACHE_PREFIX_ENV_NAME, } from "../../api/overrides/incremental-cache/kv-incremental-cache.js";
 import { BINDING_NAME as R2_CACHE_BINDING_NAME, NAME as R2_CACHE_NAME, PREFIX_ENV_NAME as R2_CACHE_PREFIX_ENV_NAME, } from "../../api/overrides/incremental-cache/r2-incremental-cache.js";
@@ -103,18 +105,112 @@ export function getCacheAssets(opts) {
     }
     return assets;
 }
-async function populateR2IncrementalCache(buildOpts, config, populateCacheOptions, envVars) {
-    logger.info("\nPopulating R2 incremental cache...");
-    const binding = config.r2_buckets.find(({ binding }) => binding === R2_CACHE_BINDING_NAME);
-    if (!binding) {
-        throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`);
+/**
+ * Create a temporary configuration file for batch upload from environment variables
+ * @returns Path to the temporary config file or null if env vars not available
+ */
+function createTempRcloneConfig(accessKey, secretKey, accountId) {
+    const tempDir = tmpdir();
+    const tempConfigPath = path.join(tempDir, `rclone-config-${Date.now()}.conf`);
+    const configContent = `[r2]
+type = s3
+provider = Cloudflare
+access_key_id = ${accessKey}
+secret_access_key = ${secretKey}
+endpoint = https://${accountId}.r2.cloudflarestorage.com
+acl = private
+`;
+    /**
+     * 0o600 is an octal number (the 0o prefix indicates octal in JavaScript)
+     * that represents Unix file permissions:
+     *
+     * - 6 (owner): read (4) + write (2) = readable and writable by the file owner
+     * - 0 (group): no permissions for the group
+     * - 0 (others): no permissions for anyone else
+     *
+     * In symbolic notation, this is: rw-------
+     */
+    writeFileSync(tempConfigPath, configContent, { mode: 0o600 });
+    return tempConfigPath;
+}
+/**
+ * Populate R2 incremental cache using batch upload for better performance
+ * Uses parallel transfers to significantly speed up cache population
+ */
+async function populateR2IncrementalCacheWithBatchUpload(bucket, prefix, assets, envVars) {
+    const accessKey = envVars.R2_ACCESS_KEY_ID || null;
+    const secretKey = envVars.R2_SECRET_ACCESS_KEY || null;
+    const accountId = envVars.CF_ACCOUNT_ID || null;
+    // Ensure all required env vars are set correctly
+    if (!accessKey || !secretKey || !accountId) {
+        throw new Error("Please set R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and CF_ACCOUNT_ID environment variables to enable faster batch upload for remote R2.");
     }
-    const bucket = binding.bucket_name;
-    if (!bucket) {
-        throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`);
+    logger.info("\nPopulating remote R2 incremental cache using batch upload...");
+    // Create temporary config from env vars - required for batch upload
+    const tempConfigPath = createTempRcloneConfig(accessKey, secretKey, accountId);
+    if (!tempConfigPath) {
+        throw new Error("Failed to create temporary rclone config for R2 batch upload.");
     }
-    const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME];
-    const assets = getCacheAssets(buildOpts);
+    const env = {
+        ...process.env,
+        RCLONE_CONFIG: tempConfigPath,
+    };
+    logger.info("Using batch upload with R2 credentials from environment variables");
+    // Create a staging dir in temp directory with proper key paths
+    const tempDir = tmpdir();
+    const stagingDir = path.join(tempDir, `.r2-staging-${Date.now()}`);
+    // Track success to ensure cleanup happens correctly
+    let success = null;
+    try {
+        mkdirSync(stagingDir, { recursive: true });
+        for (const { fullPath, key, buildId, isFetch } of assets) {
+            const cacheKey = computeCacheKey(key, {
+                prefix,
+                buildId,
+                cacheType: isFetch ? "fetch" : "cache",
+            });
+            const destPath = path.join(stagingDir, cacheKey);
+            mkdirSync(path.dirname(destPath), { recursive: true });
+            copyFileSync(fullPath, destPath);
+        }
+        // Use rclone.js to sync the R2
+        const remote = `r2:${bucket}`;
+        // Using rclone.js Promise-based API for the copy operation
+        await rclone.promises.copy(stagingDir, remote, {
+            progress: true,
+            transfers: 16,
+            checkers: 8,
+            env,
+        });
+        logger.info(`Successfully uploaded ${assets.length} assets to R2 using batch upload`);
+        success = true;
+    }
+    finally {
+        try {
+            // Cleanup temporary staging directory
+            rmSync(stagingDir, { recursive: true, force: true });
+        }
+        catch {
+            console.warn(`Failed to remove temporary staging directory at ${stagingDir}`);
+        }
+        try {
+            // Cleanup temporary config file
+            rmSync(tempConfigPath);
+        }
+        catch {
+            console.warn(`Failed to remove temporary config at ${tempConfigPath}`);
+        }
+    }
+    if (!success) {
+        throw new Error("R2 batch upload failed, falling back to sequential uploads...");
+    }
+}
+/**
+ * Populate R2 incremental cache using sequential Wrangler uploads
+ * Falls back to this method when batch upload is not available or fails
+ */
+async function populateR2IncrementalCacheWithSequentialUpload(buildOpts, bucket, prefix, assets, populateCacheOptions) {
+    logger.info("Using sequential cache uploads.");
     for (const { fullPath, key, buildId, isFetch } of tqdm(assets)) {
         const cacheKey = computeCacheKey(key, {
             prefix,
@@ -136,6 +232,34 @@ async function populateR2IncrementalCache(buildOpts, config, populateCacheOption
     }
     logger.info(`Successfully populated cache with ${assets.length} assets`);
 }
+async function populateR2IncrementalCache(buildOpts, config, populateCacheOptions, envVars) {
+    logger.info("\nPopulating R2 incremental cache...");
+    const binding = config.r2_buckets.find(({ binding }) => binding === R2_CACHE_BINDING_NAME);
+    if (!binding) {
+        throw new Error(`No R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} found!`);
+    }
+    const bucket = binding.bucket_name;
+    if (!bucket) {
+        throw new Error(`R2 binding ${JSON.stringify(R2_CACHE_BINDING_NAME)} should have a 'bucket_name'`);
+    }
+    const prefix = envVars[R2_CACHE_PREFIX_ENV_NAME];
+    const assets = getCacheAssets(buildOpts);
+    // Force sequential upload for local target
+    if (populateCacheOptions.target === "local") {
+        logger.info("Using sequential upload for local R2 (batch upload only works with remote R2)");
+        return await populateR2IncrementalCacheWithSequentialUpload(buildOpts, bucket, prefix, assets, populateCacheOptions);
+    }
+    try {
+        // Attempt batch upload first (using rclone) - only for remote target
+        return await populateR2IncrementalCacheWithBatchUpload(bucket, prefix, assets, envVars);
+    }
+    catch (error) {
+        logger.warn(`Batch upload failed: ${error instanceof Error ? error.message : error}`);
+        logger.info("Falling back to sequential uploads...");
+        // Sequential upload fallback (using Wrangler)
+        return await populateR2IncrementalCacheWithSequentialUpload(buildOpts, bucket, prefix, assets, populateCacheOptions);
+    }
+}
 async function populateKVIncrementalCache(buildOpts, config, populateCacheOptions, envVars) {
     logger.info("\nPopulating KV incremental cache...");
     const binding = config.kv_namespaces.find(({ binding }) => binding === KV_CACHE_BINDING_NAME);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@opennextjs/cloudflare",
   "description": "Cloudflare builder for next apps",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "type": "module",
   "bin": {
     "opennextjs-cloudflare": "dist/cli/index.js"
@@ -43,10 +43,12 @@
   "homepage": "https://github.com/opennextjs/opennextjs-cloudflare",
   "dependencies": {
     "@dotenvx/dotenvx": "1.31.0",
-    "@opennextjs/aws": "3.8.2",
+    "@opennextjs/aws": "3.8.5",
+    "@types/rclone.js": "^0.6.3",
     "cloudflare": "^4.4.1",
     "enquirer": "^2.4.1",
     "glob": "^11.0.0",
+    "rclone.js": "^0.6.6",
     "ts-tqdm": "^0.8.6",
     "yargs": "^18.0.0"
   },