@opennextjs/cloudflare 1.6.0 → 1.6.2

package/dist/api/durable-objects/sharded-tag-cache.d.ts

@@ -5,4 +5,5 @@ export declare class DOShardedTagCache extends DurableObject<CloudflareEnv> {
     getLastRevalidated(tags: string[]): Promise<number>;
     hasBeenRevalidated(tags: string[], lastModified?: number): Promise<boolean>;
     writeTags(tags: string[], lastModified: number): Promise<void>;
+    getRevalidationTimes(tags: string[]): Promise<Record<string, number>>;
 }

package/dist/api/durable-objects/sharded-tag-cache.js

@@ -34,4 +34,10 @@ export class DOShardedTagCache extends DurableObject {
             this.sql.exec(`INSERT OR REPLACE INTO revalidations (tag, revalidatedAt) VALUES (?, ?)`, tag, lastModified);
         });
     }
+    async getRevalidationTimes(tags) {
+        const result = this.sql
+            .exec(`SELECT tag, revalidatedAt FROM revalidations WHERE tag IN (${tags.map(() => "?").join(", ")})`, ...tags)
+            .toArray();
+        return Object.fromEntries(result.map((row) => [row.tag, row.revalidatedAt]));
+    }
 }

package/dist/api/overrides/tag-cache/do-sharded-tag-cache.d.ts

@@ -1,4 +1,5 @@
 import type { NextModeTagCache } from "@opennextjs/aws/types/overrides.js";
+import { DOShardedTagCache } from "../../durable-objects/sharded-tag-cache.js";
 export declare const DEFAULT_WRITE_RETRIES = 3;
 export declare const DEFAULT_NUM_SHARDS = 4;
 export declare const NAME = "do-sharded-tag-cache";
@@ -34,6 +35,13 @@ interface ShardedDOTagCacheOptions {
      * @default 5
      */
     regionalCacheTtlSec?: number;
+    /**
+     * Whether to persist missing tags in the regional cache.
+     * This is dangerous if you don't invalidate the Cache API when you revalidate tags as you could end up storing stale data in the data cache.
+     *
+     * @default false
+     */
+    regionalCacheDangerouslyPersistMissingTags?: boolean;
     /**
      * Enable shard replication to handle higher load.
      *
@@ -91,7 +99,6 @@ export declare class DOId {
 interface CacheTagKeyOptions {
     doId: DOId;
     tags: string[];
-    type: "boolean" | "number";
 }
 declare class ShardedDOTagCache implements NextModeTagCache {
     private opts;
@@ -145,9 +152,21 @@ declare class ShardedDOTagCache implements NextModeTagCache {
     writeTags(tags: string[]): Promise<void>;
     performWriteTagsWithRetry(doId: DOId, tags: string[], lastModified: number, retryNumber?: number): Promise<void>;
     getCacheInstance(): Promise<Cache | undefined>;
-    getCacheUrlKey(opts: CacheTagKeyOptions): string;
-    getFromRegionalCache(opts: CacheTagKeyOptions): Promise<Response | undefined>;
-    putToRegionalCache(optsKey: CacheTagKeyOptions, value: number | boolean): Promise<void>;
+    getCacheUrlKey(doId: DOId, tag: string): string;
+    /**
+     * Get the last revalidation time for the tags from the regional cache
+     * If the cache is not enabled, it will return an empty array
+     * @returns An array of objects with the tag and the last revalidation time
+     */
+    getFromRegionalCache(opts: CacheTagKeyOptions): Promise<{
+        tag: string;
+        time: number;
+    }[]>;
+    putToRegionalCache(optsKey: CacheTagKeyOptions, stub: DurableObjectStub<DOShardedTagCache>): Promise<void>;
+    /**
+     * Deletes the regional cache for the given tags
+     * This is used to ensure that the cache is cleared when the tags are revalidated
+     */
     deleteRegionalCache(optsKey: CacheTagKeyOptions): Promise<void>;
 }
 declare const _default: (opts?: ShardedDOTagCacheOptions) => ShardedDOTagCache;

package/dist/api/overrides/tag-cache/do-sharded-tag-cache.js

@@ -179,26 +179,25 @@ class ShardedDOTagCache {
         const { isDisabled } = await this.getConfig();
         if (isDisabled)
             return 0;
+        if (tags.length === 0)
+            return 0; // No tags to check
+        const deduplicatedTags = Array.from(new Set(tags)); // We deduplicate the tags to avoid unnecessary requests
         try {
-            const shardedTagGroups = this.groupTagsByDO({ tags });
+            const shardedTagGroups = this.groupTagsByDO({ tags: deduplicatedTags });
             const shardedTagRevalidationOutcomes = await Promise.all(shardedTagGroups.map(async ({ doId, tags }) => {
-                const cachedValue = await this.getFromRegionalCache({ doId, tags, type: "number" });
-                if (cachedValue) {
-                    const cached = await cachedValue.text();
-                    try {
-                        return parseInt(cached, 10);
-                    }
-                    catch (e) {
-                        debug("Error while parsing cached value", e);
-                        // If we can't parse the cached value, we should just ignore it and go to the durable object
-                    }
+                const cachedValue = await this.getFromRegionalCache({ doId, tags });
+                // If all the value were found in the regional cache, we can just return the max value
+                if (cachedValue.length === tags.length) {
+                    return Math.max(...cachedValue.map((item) => item.time));
                 }
+                // Otherwise we need to check the durable object on the ones that were not found in the cache
+                const filteredTags = deduplicatedTags.filter((tag) => !cachedValue.some((item) => item.tag === tag));
                 const stub = this.getDurableObjectStub(doId);
-                const _lastRevalidated = await stub.getLastRevalidated(tags);
-                if (!_lastRevalidated) {
-                    getCloudflareContext().ctx.waitUntil(this.putToRegionalCache({ doId, tags, type: "number" }, _lastRevalidated));
-                }
-                return _lastRevalidated;
+                const lastRevalidated = await stub.getLastRevalidated(filteredTags);
+                const result = Math.max(...cachedValue.map((item) => item.time), lastRevalidated);
+                // We then need to populate the regional cache with the missing tags
+                getCloudflareContext().ctx.waitUntil(this.putToRegionalCache({ doId, tags }, stub));
+                return result;
             }));
             return Math.max(...shardedTagRevalidationOutcomes);
         }
@@ -221,17 +220,20 @@ class ShardedDOTagCache {
         try {
             const shardedTagGroups = this.groupTagsByDO({ tags });
             const shardedTagRevalidationOutcomes = await Promise.all(shardedTagGroups.map(async ({ doId, tags }) => {
-                const cachedValue = await this.getFromRegionalCache({ doId, tags, type: "boolean" });
-                if (cachedValue) {
-                    return (await cachedValue.text()) === "true";
+                const cachedValue = await this.getFromRegionalCache({ doId, tags });
+                // If one of the cached values is newer than the lastModified, we can return true
+                const cacheHasBeenRevalidated = cachedValue.some((cachedValue) => {
+                    return (cachedValue.time ?? 0) > (lastModified ?? Date.now());
+                });
+                if (cacheHasBeenRevalidated) {
+                    return true;
                 }
                 const stub = this.getDurableObjectStub(doId);
                 const _hasBeenRevalidated = await stub.hasBeenRevalidated(tags, lastModified);
-                //TODO: Do we want to cache the result if it has been revalidated ?
-                // If we do so, we risk causing cache MISS even though it has been revalidated elsewhere
-                // On the other hand revalidating a tag that is used in a lot of places will cause a lot of requests
-                if (!_hasBeenRevalidated) {
-                    getCloudflareContext().ctx.waitUntil(this.putToRegionalCache({ doId, tags, type: "boolean" }, _hasBeenRevalidated));
+                const remainingTags = tags.filter((tag) => !cachedValue.some((item) => item.tag === tag));
+                if (remainingTags.length > 0) {
+                    // We need to put the missing tags in the regional cache
+                    getCloudflareContext().ctx.waitUntil(this.putToRegionalCache({ doId, tags: remainingTags }, stub));
                 }
                 return _hasBeenRevalidated;
             }));
@@ -266,10 +268,7 @@ class ShardedDOTagCache {
             await stub.writeTags(tags, lastModified);
             // Depending on the shards and the tags, deleting from the regional cache will not work for every tag
             // We also need to delete both cache
-            await Promise.all([
-                this.deleteRegionalCache({ doId, tags, type: "boolean" }),
-                this.deleteRegionalCache({ doId, tags, type: "number" }),
-            ]);
+            await Promise.all([this.deleteRegionalCache({ doId, tags })]);
         }
         catch (e) {
             error("Error while writing tags", e);
@@ -293,41 +292,79 @@ class ShardedDOTagCache {
         }
         return this.localCache;
     }
-    getCacheUrlKey(opts) {
-        const { doId, tags, type } = opts;
-        return `http://local.cache/shard/${doId.shardId}?type=${type}&tags=${encodeURIComponent(tags.join(";"))}`;
+    getCacheUrlKey(doId, tag) {
+        return `http://local.cache/shard/${doId.shardId}?tag=${encodeURIComponent(tag)}`;
     }
+    /**
+     * Get the last revalidation time for the tags from the regional cache
+     * If the cache is not enabled, it will return an empty array
+     * @returns An array of objects with the tag and the last revalidation time
+     */
     async getFromRegionalCache(opts) {
         try {
             if (!this.opts.regionalCache)
-                return;
+                return [];
             const cache = await this.getCacheInstance();
             if (!cache)
-                return;
-            return cache.match(this.getCacheUrlKey(opts));
+                return [];
+            const result = await Promise.all(opts.tags.map(async (tag) => {
+                const cachedResponse = await cache.match(this.getCacheUrlKey(opts.doId, tag));
+                if (!cachedResponse)
+                    return null;
+                const cachedText = await cachedResponse.text();
+                try {
+                    return { tag, time: parseInt(cachedText, 10) };
+                }
+                catch (e) {
+                    debugCache("Error while parsing cached value", e);
+                    return null;
+                }
+            }));
+            return result.filter((item) => item !== null);
         }
         catch (e) {
             error("Error while fetching from regional cache", e);
+            return [];
         }
     }
-    async putToRegionalCache(optsKey, value) {
+    async putToRegionalCache(optsKey, stub) {
         if (!this.opts.regionalCache)
             return;
         const cache = await this.getCacheInstance();
         if (!cache)
             return;
         const tags = optsKey.tags;
-        await cache.put(this.getCacheUrlKey(optsKey), new Response(`${value}`, {
-            headers: {
-                "cache-control": `max-age=${this.opts.regionalCacheTtlSec ?? 5}`,
-                ...(tags.length > 0
-                    ? {
-                        "cache-tag": tags.join(","),
-                    }
-                    : {}),
-            },
+        const tagsLastRevalidated = await stub.getRevalidationTimes(tags);
+        await Promise.all(tags.map(async (tag) => {
+            let lastRevalidated = tagsLastRevalidated[tag];
+            if (lastRevalidated === undefined) {
+                if (this.opts.regionalCacheDangerouslyPersistMissingTags) {
+                    lastRevalidated = 0; // If the tag is not found, we set it to 0 as it means it has never been revalidated before.
+                }
+                else {
+                    debugCache("Tag not found in revalidation times", { tag, optsKey });
+                    return; // If the tag is not found, we skip it
+                }
+            }
+            const cacheKey = this.getCacheUrlKey(optsKey.doId, tag);
+            debugCache("Putting to regional cache", { cacheKey, lastRevalidated });
+            await cache.put(cacheKey, new Response(lastRevalidated.toString(), {
+                status: 200,
+                headers: {
+                    "cache-control": `max-age=${this.opts.regionalCacheTtlSec ?? 5}`,
+                    ...(tags.length > 0
+                        ? {
+                            "cache-tag": tags.join(","),
+                        }
+                        : {}),
+                },
+            }));
         }));
     }
+    /**
+     * Deletes the regional cache for the given tags
+     * This is used to ensure that the cache is cleared when the tags are revalidated
+     */
     async deleteRegionalCache(optsKey) {
         // We never want to crash because of the cache
         try {
@@ -336,7 +373,11 @@ class ShardedDOTagCache {
             const cache = await this.getCacheInstance();
             if (!cache)
                 return;
-            await cache.delete(this.getCacheUrlKey(optsKey));
+            await Promise.all(optsKey.tags.map(async (tag) => {
+                const cacheKey = this.getCacheUrlKey(optsKey.doId, tag);
+                debugCache("Deleting from regional cache", { cacheKey });
+                await cache.delete(cacheKey);
+            }));
         }
         catch (e) {
             debugCache("Error while deleting from regional cache", e);

package/dist/cli/build/open-next/createServerBundle.js

@@ -154,6 +154,7 @@ async function generateBundle(name, options, fnOptions, codeCustomization) {
     const isAfter141 = buildHelper.compareSemver(options.nextVersion, ">=", "14.1");
     const isAfter142 = buildHelper.compareSemver(options.nextVersion, ">=", "14.2");
     const isAfter152 = buildHelper.compareSemver(options.nextVersion, ">=", "15.2.0");
+    const isAfter154 = buildHelper.compareSemver(options.nextVersion, ">=", "15.4.0");
     const disableRouting = isBefore13413 || config.middleware?.external;
     const plugins = [
         openNextReplacementPlugin({
@@ -164,6 +165,7 @@ async function generateBundle(name, options, fnOptions, codeCustomization) {
                 ...(disableRouting ? ["withRouting"] : []),
                 ...(isAfter142 ? ["patchAsyncStorage"] : []),
                 ...(isAfter141 ? ["appendPrefetch"] : []),
+                ...(isAfter154 ? [] : ["setInitialURL"]),
             ],
         }),
         openNextReplacementPlugin({

package/dist/cli/build/utils/ensure-cf-config.js

@@ -5,6 +5,8 @@ import logger from "@opennextjs/aws/logger.js";
  * @param config OpenNext configuration.
  */
 export function ensureCloudflareConfig(config) {
+    const mwIsMiddlewareExternal = config.middleware?.external === true;
+    const mwConfig = mwIsMiddlewareExternal ? config.middleware : undefined;
     const requirements = {
         // Check for the default function
         dftUseCloudflareWrapper: config.default?.override?.wrapper === "cloudflare-node",
@@ -18,10 +20,10 @@ export function ensureCloudflareConfig(config) {
             config.default?.override?.queue === "direct" ||
             typeof config.default?.override?.queue === "function",
         // Check for the middleware function
-        mwIsMiddlewareExternal: config.middleware?.external === true,
-        mwUseCloudflareWrapper: config.middleware?.override?.wrapper === "cloudflare-edge",
-        mwUseEdgeConverter: config.middleware?.override?.converter === "edge",
-        mwUseFetchProxy: config.middleware?.override?.proxyExternalRequest === "fetch",
+        mwIsMiddlewareExternal,
+        mwUseCloudflareWrapper: mwConfig?.override?.wrapper === "cloudflare-edge",
+        mwUseEdgeConverter: mwConfig?.override?.converter === "edge",
+        mwUseFetchProxy: mwConfig?.override?.proxyExternalRequest === "fetch",
         hasCryptoExternal: config.edgeExternals?.includes("node:crypto"),
     };
     if (config.default?.override?.queue === "direct") {

package/dist/cli/commands/populate-cache.js

@@ -74,10 +74,14 @@ async function populateR2IncrementalCache(options, config, populateCacheOptions)
             "r2 object put",
             quoteShellMeta(normalizePath(path.join(bucket, cacheKey))),
             `--file ${quoteShellMeta(fullPath)}`,
-        ], 
-        // NOTE: R2 does not support the environment flag and results in the following error:
-        // Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'.
-        { target: populateCacheOptions.target, logging: "error" });
+        ], {
+            target: populateCacheOptions.target,
+            configPath: populateCacheOptions.configPath,
+            // R2 does not support the environment flag and results in the following error:
+            // Incorrect type for the 'cacheExpiry' field on 'HttpMetadata': the provided value is not of type 'date'.
+            environment: undefined,
+            logging: "error",
+        });
     }
     logger.info(`Successfully populated cache with ${assets.length} assets`);
 }
@@ -107,7 +111,9 @@ async function populateKVIncrementalCache(options, config, populateCacheOptions)
         }));
         writeFileSync(chunkPath, JSON.stringify(kvMapping));
         runWrangler(options, ["kv bulk put", quoteShellMeta(chunkPath), `--binding ${KV_CACHE_BINDING_NAME}`], {
-            ...populateCacheOptions,
+            target: populateCacheOptions.target,
+            environment: populateCacheOptions.environment,
+            configPath: populateCacheOptions.configPath,
             logging: "error",
         });
         rmSync(chunkPath);
@@ -124,7 +130,12 @@ function populateD1TagCache(options, config, populateCacheOptions) {
         "d1 execute",
         D1_TAG_BINDING_NAME,
         `--command "CREATE TABLE IF NOT EXISTS revalidations (tag TEXT NOT NULL, revalidatedAt INTEGER NOT NULL, UNIQUE(tag) ON CONFLICT REPLACE);"`,
-    ], { ...populateCacheOptions, logging: "error" });
+    ], {
+        target: populateCacheOptions.target,
+        environment: populateCacheOptions.environment,
+        configPath: populateCacheOptions.configPath,
+        logging: "error",
+    });
     logger.info("\nSuccessfully created D1 table");
 }
 function populateStaticAssetsIncrementalCache(options) {

package/dist/cli/commands/skew-protection.d.ts

@@ -28,10 +28,10 @@ import type { WorkerEnvVar } from "./helpers.js";
  *
  * @param options Build options
  * @param config OpenNext config
- * @param envVars Environment variables
+ * @param workerEnvVars Worker Environment variables (taken from the wrangler config files)
  * @returns Deployment mapping or undefined
  */
-export declare function getDeploymentMapping(options: BuildOptions, config: OpenNextConfig, envVars: WorkerEnvVar): Promise<Record<string, string> | undefined>;
+export declare function getDeploymentMapping(options: BuildOptions, config: OpenNextConfig, workerEnvVars: WorkerEnvVar): Promise<Record<string, string> | undefined>;
 /**
  * Update an existing deployment mapping:
  * - Replace the "current" version with the latest deployed version

package/dist/cli/commands/skew-protection.js

@@ -36,13 +36,17 @@ const MS_PER_DAY = 24 * 3600 * 1000;
  *
  * @param options Build options
  * @param config OpenNext config
- * @param envVars Environment variables
+ * @param workerEnvVars Worker Environment variables (taken from the wrangler config files)
  * @returns Deployment mapping or undefined
  */
-export async function getDeploymentMapping(options, config, envVars) {
+export async function getDeploymentMapping(options, config, workerEnvVars) {
     if (config.cloudflare?.skewProtection?.enabled !== true) {
         return undefined;
     }
+    // Note that `process.env` is spread after `workerEnvVars` since we do want
+    // system environment variables to take precedence over the variables defined
+    // in the wrangler config files
+    const envVars = { ...workerEnvVars, ...process.env };
     const nextConfig = loadConfig(path.join(options.appBuildOutputPath, ".next"));
     const deploymentId = nextConfig.deploymentId;
     if (!deploymentId) {

package/dist/cli/utils/run-wrangler.d.ts

@@ -3,6 +3,7 @@ export type WranglerTarget = "local" | "remote";
 type WranglerOptions = {
     target?: WranglerTarget;
     environment?: string;
+    configPath?: string;
     logging?: "all" | "error";
 };
 export declare function runWrangler(options: BuildOptions, args: string[], wranglerOpts?: WranglerOptions): void;

package/dist/cli/utils/run-wrangler.js

@@ -43,6 +43,7 @@ export function runWrangler(options, args, wranglerOpts = {}) {
         ...injectPassthroughFlagForArgs(options, [
             ...args,
             wranglerOpts.environment && `--env ${wranglerOpts.environment}`,
+            wranglerOpts.configPath && `--config ${wranglerOpts.configPath}`,
             wranglerOpts.target === "remote" && "--remote",
             wranglerOpts.target === "local" && "--local",
         ].filter((v) => !!v)),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@opennextjs/cloudflare",
   "description": "Cloudflare builder for next apps",
-  "version": "1.6.0",
+  "version": "1.6.2",
   "type": "module",
   "bin": {
     "opennextjs-cloudflare": "dist/cli/index.js"
@@ -43,7 +43,7 @@
   "homepage": "https://github.com/opennextjs/opennextjs-cloudflare",
   "dependencies": {
     "@dotenvx/dotenvx": "1.31.0",
-    "@opennextjs/aws": "3.7.0",
+    "@opennextjs/aws": "3.7.1",
     "cloudflare": "^4.4.1",
     "enquirer": "^2.4.1",
     "glob": "^11.0.0",