@trieb.work/nextjs-turbo-redis-cache 1.15.0 → 1.15.1

package/src/CacheComponentsHandler.ts

@@ -6,6 +6,7 @@ import {
   redisErrorHandler,
 } from './RedisStringsHandler';
 import { SyncedMap } from './SyncedMap';
+import { DeduplicatedRequestHandler } from './DeduplicatedRequestHandler';
 import { debug } from './utils/debug';
 import { resolveKeyPrefix } from './utils/prefix';
 
@@ -87,8 +88,19 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
   private client: Client;
   private revalidatedTagsMap: SyncedMap<number>;
   private sharedTagsMap: SyncedMap<string[]>;
+  private inMemoryDeduplicationCache: SyncedMap<
+    Promise<ReturnType<Client['get']>>
+  >;
   private keyPrefix: string;
   private getTimeoutMs: number;
+  private redisGet: Client['get'];
+  private redisDeduplicationHandler: DeduplicatedRequestHandler<
+    Client['get'],
+    string | Buffer | null
+  >;
+  private deduplicatedRedisGet: (key: string) => Client['get'];
+  private redisGetDeduplication: boolean;
+  private inMemoryCachingTime: number;
 
   constructor({
     redisUrl = process.env.REDIS_URL
@@ -103,6 +115,8 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
       : 500,
     revalidateTagQuerySize = 250,
     avgResyncIntervalMs = 60 * 60 * 1_000,
+    redisGetDeduplication = true,
+    inMemoryCachingTime = 10_000,
     socketOptions,
     clientOptions,
     killContainerOnErrorThreshold = process.env
@@ -118,6 +132,8 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
         env: process.env,
       });
       this.getTimeoutMs = getTimeoutMs;
+      this.redisGetDeduplication = redisGetDeduplication;
+      this.inMemoryCachingTime = inMemoryCachingTime;
 
       this.client = createClient({
         url: redisUrl,
@@ -208,6 +224,29 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
           avgResyncIntervalMs / 10 +
           Math.random() * (avgResyncIntervalMs / 10),
       });
+
+      this.inMemoryDeduplicationCache = new SyncedMap({
+        client: this.client,
+        keyPrefix: this.keyPrefix,
+        redisKey: '__cacheComponents_inMemoryDeduplicationCache__',
+        database,
+        querySize: revalidateTagQuerySize,
+        filterKeys,
+        customizedSync: {
+          withoutRedisHashmap: true,
+          withoutSetSync: true,
+        },
+      });
+
+      const redisGet: Client['get'] = this.client.get.bind(this.client);
+      this.redisDeduplicationHandler = new DeduplicatedRequestHandler(
+        redisGet,
+        inMemoryCachingTime,
+        this.inMemoryDeduplicationCache,
+      );
+      this.redisGet = redisGet;
+      this.deduplicatedRedisGet =
+        this.redisDeduplicationHandler.deduplicatedFunction;
     } catch (error) {
       console.error('RedisCacheComponentsHandler constructor error', error);
       throw error;
@@ -253,12 +292,18 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
     try {
       await this.assertClientIsReady();
 
+      const clientGet = this.redisGetDeduplication
+        ? this.deduplicatedRedisGet(cacheKey)
+        : this.redisGet;
+
       const serialized = await redisErrorHandler(
-        'RedisCacheComponentsHandler.get(), operation: get ' +
+        'RedisCacheComponentsHandler.get(), operation: get' +
+          (this.redisGetDeduplication ? ' deduplicated' : '') +
+          ' ' +
           this.getTimeoutMs +
           'ms ' +
           redisKey,
-        this.client.get(
+        clientGet(
           commandOptions({ signal: AbortSignal.timeout(this.getTimeoutMs) }),
           redisKey,
         ),
@@ -355,6 +400,10 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
         throw jsonError;
       }
 
+      if (this.redisGetDeduplication) {
+        this.redisDeduplicationHandler.seedRequestReturn(cacheKey, serialized);
+      }
+
       // expire is already a duration in seconds, use it directly
       const ttlSeconds =
         Number.isFinite(stored.expire) && stored.expire > 0
@@ -450,6 +499,12 @@ class RedisCacheComponentsHandler implements CacheComponentsHandler {
         this.client.unlink(fullRedisKeys),
       );
 
+      if (this.redisGetDeduplication && this.inMemoryCachingTime > 0) {
+        for (const key of keysToDelete) {
+          this.inMemoryDeduplicationCache.delete(key);
+        }
+      }
+
       // Delete from sharedTagsMap
       const deleteTagsOperation = this.sharedTagsMap.delete(cacheKeys);
       await deleteTagsOperation;

package/src/SyncedMap.ts

@@ -41,6 +41,7 @@ export class SyncedMap<V> {
 
   private setupLock: Promise<void>;
   private setupLockResolve!: () => void;
+  private isReconnecting = false;
 
   constructor(options: SyncedMapOptions) {
     this.client = options.client;
@@ -209,6 +210,16 @@ export class SyncedMap<V> {
     };
 
     try {
+      // Attach error handler BEFORE subscribing so that errors during
+      // subscribe or after connection loss are always caught and trigger
+      // a reconnection. Previously this was attached after the subscribe()
+      // calls, which meant if subscribe threw during an outage the handler
+      // was never attached and the subscriber was permanently dead.
+      this.subscriberClient.on('error', async (err) => {
+        console.error('Subscriber client error:', err);
+        await this.reconnectSubscriber();
+      });
+
       const connectIfNeeded = async () => {
         // Avoid overlapping connect() calls which can lead to
         // "Socket already opened" errors when a connect is already in progress
@@ -241,7 +252,9 @@ export class SyncedMap<V> {
           throw new Error(
             'Keyspace event configuration is set to "' +
               keyspaceEventConfig +
-              "\" but has to include 'E' for Keyevent events, published with __keyevent@<db>__ prefix. We recommend to set it to 'Exe' like so `redis-cli -h localhost config set notify-keyspace-events Exe`",
+              " but has to include 'E' for Keyevent events, " +
+              'published with __keyevent@<db>__ prefix. We recommend to set it to ' +
+              "'Exe' like so `redis-cli -h localhost config set notify-keyspace-events Exe`",
           );
         }
         if (
@@ -254,7 +267,9 @@ export class SyncedMap<V> {
           throw new Error(
             'Keyspace event configuration is set to "' +
               keyspaceEventConfig +
-              "\" but has to include 'A' or 'x' and 'e' for expired and evicted events. We recommend to set it to 'Exe' like so `redis-cli -h localhost config set notify-keyspace-events Exe`",
+              " but has to include 'A' or 'x' and 'e' for expired and " +
+              'evicted events. We recommend to set it to ' +
+              "'Exe' like so `redis-cli -h localhost config set notify-keyspace-events Exe`",
           );
         }
       }
@@ -275,24 +290,58 @@ export class SyncedMap<V> {
           keyEventHandler,
         ),
       ]);
+    } catch (error) {
+      console.error('Error setting up pub/sub client:', error);
+      throw error;
+    }
+  }
 
-      // Error handling for reconnection
-      this.subscriberClient.on('error', async (err) => {
-        console.error('Subscriber client error:', err);
+  /**
+   * Reconnect the subscriber client with exponential backoff.
+   * Guarded by `isReconnecting` to prevent concurrent reconnection
+   * attempts when multiple error events fire during an outage.
+   * Retries indefinitely (with capped delay) so the subscriber
+   * eventually recovers once Redis is available again.
+   */
+  private async reconnectSubscriber(): Promise<void> {
+    if (this.isReconnecting) return;
+    this.isReconnecting = true;
+
+    try {
+      let attempt = 0;
+      while (true) {
         try {
-          await this.subscriberClient.quit();
+          // Forcibly tear down the old subscriber so its internal reconnect
+          // loop cannot come back to life once Redis recovers. `quit()` waits
+          // for the server and can silently fail during an outage, leaving a
+          // zombie client that still fires our error handler. Remove that
+          // handler first so a delayed error from the old socket cannot tear
+          // down the replacement subscriber we are about to create.
+          try {
+            this.subscriberClient.removeAllListeners('error');
+            await this.subscriberClient.disconnect();
+          } catch {
+            // expected during outage
+          }
+          // Create a fresh subscriber client
           this.subscriberClient = this.client.duplicate();
           await this.setupPubSub();
-        } catch (reconnectError) {
+          return; // success
+        } catch (error) {
+          const delay = Math.min(
+            500 * Math.pow(2, Math.min(attempt, 5)),
+            10_000,
+          );
+          attempt++;
           console.error(
-            'Failed to reconnect subscriber client:',
-            reconnectError,
+            `Subscriber reconnect attempt ${attempt} failed, retrying in ${delay}ms`,
+            error,
           );
+          await new Promise((r) => setTimeout(r, delay));
         }
-      });
-    } catch (error) {
-      console.error('Error setting up pub/sub client:', error);
-      throw error;
+      }
+    } finally {
+      this.isReconnecting = false;
     }
   }
 

package/test/README.md

@@ -0,0 +1,179 @@
+# Test Structure
+
+```
+test/
+├── playwright/              # E2E tests — browser automation (Playwright)
+├── vitest/                  # Unit + Integration tests (Vitest)
+│   ├── unit/                # Pure unit tests (no external deps)
+│   └── integration/         # Integration tests (Redis + Next.js server)
+│       └── cache-components/  # Cache Components integration (Next.js 16+)
+└── nextjs-test-projects/    # Next.js app fixtures (shared across test types)
+```
+
+## Overview
+
+| Layer           | Runner     | What it validates                                                       | Needs Redis? | Needs Next.js app? |
+| --------------- | ---------- | ----------------------------------------------------------------------- | ------------ | ------------------ |
+| **Unit**        | Vitest     | Logic in isolation (serializer, prefix resolution, reconnect handling)  | No           | No                 |
+| **Integration** | Vitest     | Cache handler ↔ Redis ↔ Next.js HTTP responses (server-side plumbing) | Yes          | Yes                |
+| **E2E**         | Playwright | User-facing behavior in a real browser (Cache Components / `use cache`) | Yes          | Yes                |
+
+---
+
+## Unit Tests (`test/vitest/unit/`)
+
+**Runner:** Vitest
+**Command:** `pnpm test:unit`
+**Config:** `vite.config.ts`
+
+Fast tests with no external dependencies. Mocks are used where needed.
+
+| File                                      | What it tests                                                                    |
+| ----------------------------------------- | -------------------------------------------------------------------------------- |
+| `serializer.test.ts`                      | `CacheValueSerializer` interface, JSON round-trips, singleton stability          |
+| `index.test.ts`                           | `RedisStringsHandler` constructor options, default behaviors                     |
+| `utils/prefix.test.ts`                    | `resolveKeyPrefix` logic (BUILD_ID fallback, env var precedence)                 |
+| `reconnect-socket-already-opened.test.ts` | Regression: reconnect logic doesn't call `connect()` when socket is already open |
+
+```bash
+pnpm test:unit          # single run
+pnpm test:unit:watch    # watch mode
+pnpm test:unit:coverage # with coverage report (used in CI)
+```
+
+---
+
+## Integration Tests (`test/vitest/integration/`)
+
+**Runner:** Vitest
+**Command:** `pnpm test:integration`
+**Config:** `vite.config.ts`
+**Requires:** Redis on localhost:6379, pre-built Next.js test app
+
+These tests spawn a real Next.js server as a child process, make `fetch()` requests against it, and verify both HTTP responses and Redis state directly.
+
+### Standard Integration (`nextjs-cache-handler.integration.test.ts`)
+
+Full cache lifecycle: static pages, fetch caching, revalidation, tag invalidation, TTL behavior. In CI this runs against a matrix of Next.js versions (15.4–16.2).
+
+```bash
+pnpm test:integration
+```
+
+### BUILD_ID Prefix (`build-id-prefix.integration.test.ts`)
+
+Verifies that when neither `KEY_PREFIX` nor `VERCEL_URL` is set, the handler falls back to `.next/BUILD_ID` as the Redis key prefix. Runs in its own CI job because it needs a clean environment without those env vars.
+
+```bash
+pnpm test:integration:build-id-prefix
+```
+
+### Cache Components (`cache-components/`)
+
+Integration tests specific to Next.js 16 Cache Components (`use cache`). Uses `next-app-16-2-6-cache-components` as the test app.
+
+| File                                   | What it tests                                                                                          |
+| -------------------------------------- | ------------------------------------------------------------------------------------------------------ |
+| `cache-components.integration.test.ts` | `use cache` lifecycle: store, retrieve, tag invalidation, `cacheLife` expiry                           |
+| `redis-kill-reconnect.test.ts`         | Graceful recovery when Redis drops and reconnects (main client ping only)                              |
+| `redis-subscriber-outage.test.ts`      | Issue #86: subscriber PubSub does not recover after Redis outage (expected to fail until bug is fixed) |
+| `redis-quit-vs-disconnect.test.ts`     | `quit()` does not reliably close a subscriber during an outage; `disconnect()` does                    |
+
+`redis-kill-reconnect.test.ts` uses a container runtime and auto-detects `podman` first, then `docker`. You can force runtime selection with `CONTAINER_RUNTIME=podman` or `CONTAINER_RUNTIME=docker`.
+
+```bash
+pnpm test:integration:cache-components
+```
+
+**Prerequisites for all integration tests:**
+
+```bash
+cd test/nextjs-test-projects/<app-name>
+pnpm install && pnpm build
+```
+
+---
+
+## E2E Tests (`test/playwright/`)
+
+**Runner:** Playwright
+**Command:** `pnpm test:e2e`
+**Config:** `playwright.config.ts`
+**Requires:** Redis on localhost:6379 (Playwright auto-starts `next-app-16-2-6-cache-components` via `webServer`)
+
+Browser-based tests that validate Cache Components behavior from the user's perspective. Playwright was introduced because the Cache Components (`use cache`) feature in Next.js 16 relies on interactions that `fetch()` alone cannot reproduce:
+
+- **Server Actions** are triggered by form submissions / button clicks — requires a real browser context
+- `**updateTag`\*\* must be called from within a Server Action — needs actual UI interaction to verify
+- **Cookie-based cache keys** depend on the browser sending cookies during navigation
+- **Stale-while-revalidate** effects are only observable through page reloads and DOM diffing
+
+The Vitest cache-components integration tests verify the server-side plumbing (Redis state, HTTP responses). Playwright closes the gap by testing the full user-facing flow.
+
+| File                 | What it tests                                                                                                        |
+| -------------------- | -------------------------------------------------------------------------------------------------------------------- |
+| `cache-lab.spec.ts`  | `use cache` stability, tag invalidation (`updateTag`/`revalidateTag`), runtime cookie-based cache keys, SWR behavior |
+| `update-tag.spec.ts` | `updateTag` via Server Actions (button click → action → UI update)                                                   |
+
+```bash
+pnpm test:e2e
+
+# Test against a specific Next.js version:
+PLAYWRIGHT_TEST_APP=next-app-16-0-11-cache-components pnpm test:e2e
+
+# Or point at an already-running server:
+PLAYWRIGHT_BASE_URL=http://localhost:3001 pnpm test:e2e
+```
+
+---
+
+## Next.js Test Projects (`test/nextjs-test-projects/`)
+
+Minimal Next.js applications used as fixtures. They are not test runners — they provide the server that tests run against.
+
+| App                                 | Next.js | Used by                                                        |
+| ----------------------------------- | ------- | -------------------------------------------------------------- |
+| `next-app-15-4-11`                  | 15.4.11 | Integration (matrix, default for local), build-id-prefix       |
+| `next-app-16-0-11`                  | 16.0.11 | Integration (matrix)                                           |
+| `next-app-16-2-6`                   | 16.2.6  | Integration (matrix)                                           |
+| `next-app-16-0-11-cache-components` | 16.0.11 | Integration (cache-components matrix), E2E (Playwright matrix) |
+| `next-app-16-2-6-cache-components`  | 16.2.6  | Integration (cache-components matrix), E2E (Playwright matrix) |
+| `next-app-customized`               | —       | Example of custom config (referenced in project README)        |
+
+---
+
+## CI Jobs
+
+The CI workflow (`.github/workflows/ci.yml`) is structured as:
+
+```
+lint-and-unit                        → Lint + Unit Tests + Coverage
+  ├── integration                    → Matrix: 3 Next.js versions (15.4–16.2)
+  ├── integration-build-id-prefix    → Isolated BUILD_ID prefix test
+  ├── integration-cache-components   → Matrix: 16.0.11 + 16.2.6 cache-components
+  └── e2e                            → Matrix: Playwright against 16.0.11 + 16.2.6
+```
+
+`lint-and-unit` runs first as a gate. All other jobs run in parallel after it passes.
+
+| CI Job                         | Test App(s)                                               | What runs                                                       |
+| ------------------------------ | --------------------------------------------------------- | --------------------------------------------------------------- |
+| `lint-and-unit`                | —                                                         | `pnpm lint` + `pnpm test:unit:coverage`                         |
+| `integration`                  | `next-app-15-4-11`, `next-app-16-0-11`, `next-app-16-2-6` | `pnpm test:integration` (per matrix entry)                      |
+| `integration-build-id-prefix`  | `next-app-15-4-11`                                        | `pnpm test:integration:build-id-prefix`                         |
+| `integration-cache-components` | `next-app-16-{0-3,2-3}-cache-components`                  | `pnpm test:integration:cache-components` + Redis kill/reconnect |
+| `e2e`                          | `next-app-16-{0-3,2-3}-cache-components`                  | `pnpm test:e2e` (Playwright)                                    |
+
+---
+
+## Environment Variables
+
+| Variable                | Used by                        | Description                                                               |
+| ----------------------- | ------------------------------ | ------------------------------------------------------------------------- |
+| `NEXT_TEST_APP`         | Integration                    | Which test app to use (default: `next-app-15-4-11`)                       |
+| `CACHE_COMPONENTS_APP`  | Integration (cache-components) | Which cache-components app (default: `next-app-16-2-6-cache-components`)  |
+| `PLAYWRIGHT_TEST_APP`   | E2E                            | Which app Playwright starts (default: `next-app-16-2-6-cache-components`) |
+| `PLAYWRIGHT_BASE_URL`   | E2E                            | Override base URL (skips `webServer` auto-start)                          |
+| `SKIP_BUILD`            | Integration                    | Skip Next.js build if app is pre-built                                    |
+| `DEBUG_INTEGRATION`     | Integration                    | Print child process stdout/stderr                                         |
+| `CACHE_COMPONENTS_PORT` | Integration (cache-components) | Port for the server (default: 3065)                                       |

package/test/{integration/next-app-15-0-3 → nextjs-test-projects/next-app-15-4-11}/package.json

@@ -3,27 +3,27 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev": "export VERCEL_URL=\"dev-test-$RANDOM\" && echo $VERCEL_URL && cd ../../.. && pnpm build && cd test/integration/next-app-15-0-3 && pnpm i && pnpm build && VERCEL_ENV=production REDISHOST=localhost REDISPORT=6379 DEBUG_CACHE_HANDLER=true pnpm start",
+    "dev": "export VERCEL_URL=\"dev-test-$RANDOM\" && echo $VERCEL_URL && cd ../../.. && pnpm build && cd test/nextjs-test-projects/next-app-15-4-11 && pnpm i && pnpm build && VERCEL_ENV=production REDISHOST=localhost REDISPORT=6379 DEBUG_CACHE_HANDLER=true pnpm start",
     "build": "next build",
     "start": "next start",
     "lint": "next lint"
   },
   "dependencies": {
-    "react": "19.0.0-rc-66855b96-20241106",
-    "react-dom": "19.0.0-rc-66855b96-20241106",
-    "next": "15.0.3",
+    "react": "19.2.0",
+    "react-dom": "19.2.0",
+    "next": "15.4.11",
     "redis": "4.7.0",
     "@trieb.work/nextjs-turbo-redis-cache": "file:../../../"
   },
   "devDependencies": {
     "typescript": "^5",
-    "@types/node": "^20",
+    "@types/node": "^22",
     "@types/react": "^19",
     "@types/react-dom": "^19",
     "@tailwindcss/postcss": "^4",
     "tailwindcss": "^4",
     "eslint": "^9",
-    "eslint-config-next": "15.3.0",
+    "eslint-config-next": "15.4.11",
     "@eslint/eslintrc": "^3"
   }
 }