@podosoft/podokit 1.4.1 → 1.4.2

package/dist/templates/fullstack/apps/web/src/lib/server/backend-proxy.test.ts

@@ -3,9 +3,13 @@ import type { AddressInfo } from "node:net";
 import { gzipSync } from "node:zlib";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import {
+  backendReachable,
   bodyLimitFor,
+  checkBackend,
+  envMilliseconds,
   IN_MEMORY_RESPONSE_BYTES,
   proxyRequest,
+  reportBackendAnswer,
   rawSubpath,
   resolveClientIp,
   sortedBodyLimits,
@@ -564,3 +568,174 @@ describe("backend proxy application body limits", () => {
     );
   });
 });
+
+describe("backend proxy outages", () => {
+  type Fetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
+
+  afterEach(() => {
+    reportBackendAnswer();
+    vi.unstubAllGlobals();
+    vi.unstubAllEnvs();
+  });
+
+  // Never answers on its own, like a blackholed connection or a hung API, and
+  // rejects when its signal aborts.
+  const hang: Fetch = (_input, init) =>
+    new Promise((_settle, reject) => {
+      init?.signal?.addEventListener("abort", () => reject(init.signal?.reason));
+    });
+  const refuse: Fetch = async () => {
+    throw new TypeError("Unable to connect. Is the computer able to access the url?");
+  };
+
+  // Routes the background GET /health check and the proxied request separately.
+  function stubApi(health: Fetch, upstream: Fetch): ReturnType<typeof vi.fn<Fetch>> {
+    const fetch = vi.fn<Fetch>((input, init) =>
+      String(input).endsWith("/health") ? health(input, init) : upstream(input, init),
+    );
+    vi.stubGlobal("fetch", fetch);
+    return fetch;
+  }
+
+  async function takeApiDown(): Promise<void> {
+    stubApi(refuse, refuse);
+    await checkBackend();
+    expect(backendReachable()).toBe(false);
+  }
+
+  it("answers 503 with the error envelope when the API cannot be reached", async () => {
+    stubApi(refuse, refuse);
+
+    const response = await proxyRequest(new Request("http://app.localhost/api/blog"), "http://api:5002/blog");
+
+    expect(response.status).toBe(503);
+    expect(response.headers.get("retry-after")).toBe("2");
+    expect(response.headers.get("cache-control")).toBe("no-store");
+    expect(await response.json()).toMatchObject({
+      success: false,
+      error: { code: "BACKEND_UNAVAILABLE", statusCode: 503, path: "/api/blog" },
+    });
+    await checkBackend();
+    expect(backendReachable()).toBe(false);
+  });
+
+  it("answers 504 when the API sends no response headers before the deadline", async () => {
+    vi.stubEnv("BACKEND_PROXY_TIMEOUT_MS", "20");
+    stubApi(async () => new Response("ok"), hang);
+
+    const response = await proxyRequest(new Request("http://app.localhost/api/report"), "http://api:5002/report");
+
+    expect(response.status).toBe(504);
+    expect(response.headers.get("retry-after")).toBeNull();
+    expect(await response.json()).toMatchObject({ error: { code: "BACKEND_TIMEOUT", statusCode: 504 } });
+    await checkBackend();
+    expect(backendReachable()).toBe(true);
+  });
+
+  it("fails requests still waiting for headers once the API is found unreachable", async () => {
+    vi.stubEnv("BACKEND_TIMEOUT_MS", "20");
+    stubApi(hang, hang);
+
+    const started = Date.now();
+    const response = await proxyRequest(new Request("http://app.localhost/api/blog"), "http://api:5002/blog");
+
+    expect(Date.now() - started).toBeLessThan(5_000);
+    expect(response.status).toBe(503);
+    expect(await response.json()).toMatchObject({ error: { code: "BACKEND_UNAVAILABLE" } });
+    expect(backendReachable()).toBe(false);
+  });
+
+  it("keeps waiting for a slow endpoint while the API answers its health check", async () => {
+    vi.stubEnv("BACKEND_INTERNAL_URL", "http://api:5002");
+    vi.stubEnv("BACKEND_TIMEOUT_MS", "20");
+    const fetch = stubApi(
+      async () => new Response("ok"),
+      () => new Promise((settle) => setTimeout(() => settle(Response.json({ done: true })), 80)),
+    );
+
+    const response = await proxyRequest(new Request("http://app.localhost/api/report"), "http://api:5002/report");
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ done: true });
+    expect(fetch.mock.calls.map(([input]) => String(input))).toEqual([
+      "http://api:5002/report",
+      "http://api:5002/health",
+    ]);
+    expect(backendReachable()).toBe(true);
+  });
+
+  it("answers at once without contacting the API while it is unreachable", async () => {
+    await takeApiDown();
+    const fetch = stubApi(refuse, async () => new Response(null, { status: 204 }));
+
+    const response = await proxyRequest(
+      new Request("http://app.localhost/api/blog", { method: "POST", body: "{}" }),
+      "http://api:5002/blog",
+    );
+
+    expect(response.status).toBe(503);
+    expect(fetch).not.toHaveBeenCalled();
+  });
+
+  it("checks the API again after the retry interval and recovers on any HTTP answer", async () => {
+    await takeApiDown();
+    const fetch = stubApi(async () => new Response("unavailable", { status: 500 }), refuse);
+
+    expect(backendReachable(Date.now() + 2_000)).toBe(false);
+    await checkBackend();
+
+    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(backendReachable()).toBe(true);
+  });
+
+  it("relays an error status without checking the API", async () => {
+    const fetch = stubApi(refuse, async () => Response.json({ success: false }, { status: 500 }));
+
+    const response = await proxyRequest(new Request("http://app.localhost/api/blog"), "http://api:5002/blog");
+
+    expect(response.status).toBe(500);
+    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(backendReachable()).toBe(true);
+  });
+
+  it("does not report a browser that disconnected as an outage", async () => {
+    const fetch = stubApi(refuse, hang);
+    const client = new AbortController();
+    const pending = proxyRequest(
+      new Request("http://app.localhost/api/events", { signal: client.signal }),
+      "http://api:5002/events",
+    );
+    client.abort();
+
+    await expect(pending).rejects.toBeDefined();
+    expect(fetch).toHaveBeenCalledTimes(1);
+    expect(backendReachable()).toBe(true);
+  });
+
+  it("stops the deadline once response headers arrive, so a stream keeps flowing", async () => {
+    vi.stubEnv("BACKEND_PROXY_TIMEOUT_MS", "20");
+    let upstreamSignal: AbortSignal | undefined;
+    stubApi(refuse, async (_input, init) => {
+      upstreamSignal = init?.signal ?? undefined;
+      return new Response(new ReadableStream(), { headers: { "content-type": "text/event-stream" } });
+    });
+
+    const response = await proxyRequest(new Request("http://app.localhost/api/events"), "http://api:5002/events");
+    await new Promise((settle) => setTimeout(settle, 60));
+
+    expect(response.status).toBe(200);
+    expect(upstreamSignal?.aborted).toBe(false);
+  });
+
+  it.each(["0", "-1", "1.5", "soon", "2147483648"])("rejects the invalid timeout %s", (value) => {
+    vi.stubEnv("BACKEND_TIMEOUT_MS", value);
+    expect(() => envMilliseconds("BACKEND_TIMEOUT_MS", 2_000)).toThrow("whole number of milliseconds");
+  });
+
+  it("uses the default timeout when the variable is unset or empty", () => {
+    vi.stubEnv("BACKEND_TIMEOUT_MS", "");
+    expect(envMilliseconds("BACKEND_TIMEOUT_MS", 2_000)).toBe(2_000);
+    vi.stubEnv("BACKEND_TIMEOUT_MS", "1500");
+    expect(envMilliseconds("BACKEND_TIMEOUT_MS", 2_000)).toBe(1_500);
+  });
+});

package/dist/templates/fullstack/apps/web/src/lib/server/backend-proxy.ts

@@ -218,6 +218,135 @@ export function backendBaseUrl(): string {
   return process.env.BACKEND_INTERNAL_URL ?? "http://localhost:5002";
 }
 
+/**
+ * Milliseconds one server-side API call may take, response body included, and
+ * how long the proxy waits for response headers before it checks whether the
+ * API still answers at all.
+ */
+const DEFAULT_BACKEND_TIMEOUT_MS = 2_000;
+/**
+ * Milliseconds the proxy waits for the API's response headers, matching the
+ * API's default idle timeout. Streamed bodies are not cut off once they start.
+ */
+const DEFAULT_BACKEND_PROXY_TIMEOUT_MS = 30_000;
+/** Seconds between checks while the API is unreachable, also sent as Retry-After. */
+const BACKEND_RETRY_AFTER_SECONDS = 2;
+const MAX_TIMER_MS = 2_147_483_647;
+
+/** A whole number of milliseconds from the environment, or the default when unset or empty. */
+export function envMilliseconds(name: string, fallback: number): number {
+  const raw = process.env[name]?.trim();
+  if (!raw) return fallback;
+  const value = Number(raw);
+  if (!Number.isSafeInteger(value) || value < 1 || value > MAX_TIMER_MS) {
+    throw new Error(
+      `${name} must be a whole number of milliseconds from 1 to ${MAX_TIMER_MS}: ${JSON.stringify(raw)}`,
+    );
+  }
+  return value;
+}
+
+export function backendTimeoutMs(): number {
+  return envMilliseconds("BACKEND_TIMEOUT_MS", DEFAULT_BACKEND_TIMEOUT_MS);
+}
+
+function backendProxyTimeoutMs(): number {
+  return envMilliseconds("BACKEND_PROXY_TIMEOUT_MS", DEFAULT_BACKEND_PROXY_TIMEOUT_MS);
+}
+
+// An invalid value stops the server when this module loads instead of failing
+// every request that reads it.
+backendTimeoutMs();
+backendProxyTimeoutMs();
+
+// The API counts as unreachable only when GET /health gets no HTTP answer
+// within BACKEND_TIMEOUT_MS. It is checked when a request gets no answer, or
+// when the proxy waits longer than that for response headers, so one slow
+// endpoint does not fail every request in the process. Once the API is found
+// unreachable, requests still waiting for it fail, new ones fail at once instead
+// of each waiting for a refused connection, a DNS lookup, or a hung server, and
+// a background check runs at most every BACKEND_RETRY_AFTER_SECONDS. Any HTTP
+// answer, including an error status, means the API is reachable.
+let unreachable = false;
+let answers = 0;
+let lastCheckEnded = 0;
+let pendingCheck: Promise<void> | undefined;
+let outage = new AbortController();
+
+/** Aborts when the API is found unreachable, failing the requests that still wait for it. */
+export function backendOutageSignal(): AbortSignal {
+  return outage.signal;
+}
+
+/** Check whether the API answers at all. Concurrent callers share one check. */
+export function checkBackend(): Promise<void> {
+  if (!pendingCheck) {
+    const answersBefore = answers;
+    pendingCheck = fetch(`${backendBaseUrl()}/health`, { signal: AbortSignal.timeout(backendTimeoutMs()) })
+      .then(
+        (response) => {
+          reportBackendAnswer();
+          return response.body?.cancel().catch(() => undefined);
+        },
+        () => {
+          if (answers !== answersBefore) return;
+          unreachable = true;
+          outage.abort(new DOMException("The API is unavailable", "AbortError"));
+          outage = new AbortController();
+        },
+      )
+      .finally(() => {
+        lastCheckEnded = Date.now();
+        pendingCheck = undefined;
+      });
+  }
+  return pendingCheck;
+}
+
+/** Record an HTTP answer from the API, whatever its status. */
+export function reportBackendAnswer(): void {
+  answers += 1;
+  unreachable = false;
+}
+
+/** Record a request that got no HTTP answer, and check the API unless it is already known to be down. */
+export function reportBackendFailure(): Promise<void> {
+  return unreachable ? Promise.resolve() : checkBackend();
+}
+
+/**
+ * False while the API is known to be unreachable. Such a call also starts a
+ * background check when none ran in the last BACKEND_RETRY_AFTER_SECONDS.
+ */
+export function backendReachable(now = Date.now()): boolean {
+  if (!unreachable) return true;
+  if (!pendingCheck && now - lastCheckEnded >= BACKEND_RETRY_AFTER_SECONDS * 1000) void checkBackend();
+  return false;
+}
+
+function backendFailure(path: string, statusCode: 503 | 504): Response {
+  const timedOut = statusCode === 504;
+  return Response.json(
+    {
+      success: false,
+      error: {
+        code: timedOut ? "BACKEND_TIMEOUT" : "BACKEND_UNAVAILABLE",
+        message: timedOut ? "The API did not respond in time" : "The API is unavailable",
+        statusCode,
+        path,
+        timestamp: new Date().toISOString(),
+      },
+    },
+    {
+      status: statusCode,
+      headers: {
+        "cache-control": "no-store",
+        ...(timedOut ? {} : { "retry-after": String(BACKEND_RETRY_AFTER_SECONDS) }),
+      },
+    },
+  );
+}
+
 // Normalize the resolved client IP for the forwarded header. better-auth stores
 // IPv4 as-is but anonymizes IPv6 (masks the host bits), so an IPv6 loopback would
 // be recorded as "::" — map loopback to 127.0.0.1 for a clean local value, and
@@ -252,12 +381,16 @@ export async function proxyRequest(
   const forwardedIp = normalizeClientIp(clientAddress);
   if (forwardedIp) headers.set("x-forwarded-for", forwardedIp);
 
+  const { pathname } = new URL(request.url);
+  if (!backendReachable()) {
+    request.body?.cancel().catch(() => undefined);
+    return backendFailure(pathname, 503);
+  }
   const hasBody = request.method !== "GET" && request.method !== "HEAD";
   let body: ArrayBuffer | undefined;
   if (hasBody) {
     // Preserve multipart uploads and other binary bodies byte-for-byte. Reading
     // them as text corrupts bytes that are not valid UTF-8 before forwarding.
-    const { pathname } = new URL(request.url);
     const maxBytes = bodyLimitFor(pathname);
     if (maxBytes === undefined) {
       body = await request.arrayBuffer();
@@ -267,16 +400,42 @@ export async function proxyRequest(
       body = limited;
     }
   }
-  const upstream = await fetch(targetUrl, {
-    method: request.method,
-    headers,
-    body,
-    redirect: "manual",
-    // Abort the upstream request when the browser disconnects. Without this, every
-    // closed EventSource or aborted request keeps its API socket open until Bun's
-    // fetch pool is exhausted and the web process stops serving requests.
-    signal: request.signal,
-  });
+  // The deadline, the health check, and the outage signal cover only the wait
+  // for response headers, so streamed bodies such as server-sent events are not
+  // cut off.
+  const waiting = new AbortController();
+  const deadline = setTimeout(
+    () => waiting.abort(new DOMException("The API did not respond in time", "TimeoutError")),
+    backendProxyTimeoutMs(),
+  );
+  const healthCheck = setTimeout(() => void checkBackend(), backendTimeoutMs());
+  const outage = backendOutageSignal();
+  const failOnOutage = (): void => waiting.abort(outage.reason);
+  outage.addEventListener("abort", failOnOutage);
+  let upstream: Response;
+  try {
+    upstream = await fetch(targetUrl, {
+      method: request.method,
+      headers,
+      body,
+      redirect: "manual",
+      // Abort the upstream request when the browser disconnects. Without this, every
+      // closed EventSource or aborted request keeps its API socket open until Bun's
+      // fetch pool is exhausted and the web process stops serving requests.
+      signal: AbortSignal.any([request.signal, waiting.signal]),
+    });
+  } catch (cause: unknown) {
+    // Nobody waits for an answer to a request the browser abandoned.
+    if (request.signal.aborted) throw cause;
+    if (outage.aborted) return backendFailure(pathname, 503);
+    void reportBackendFailure();
+    return backendFailure(pathname, waiting.signal.aborted ? 504 : 503);
+  } finally {
+    clearTimeout(deadline);
+    clearTimeout(healthCheck);
+    outage.removeEventListener("abort", failOnOutage);
+  }
+  reportBackendAnswer();
 
   const responseHeaders = new Headers();
   for (const name of relayedResponseHeaders) {

package/dist/templates/fullstack/apps/web/src/routes/healthz/+server.ts

@@ -0,0 +1,13 @@
+import type { RequestHandler } from "@sveltejs/kit";
+
+// Liveness and readiness of the web server itself, for the Kubernetes probes,
+// the Compose healthcheck, and the image HEALTHCHECK. It never calls the API, so
+// an API outage does not take the web server out of service.
+export const GET: RequestHandler = () =>
+  new Response("ok\n", {
+    headers: {
+      "content-type": "text/plain; charset=utf-8",
+      "cache-control": "no-store",
+      "x-robots-tag": "noindex, nofollow",
+    },
+  });

package/dist/templates/fullstack/dot-env.docker.example

@@ -12,6 +12,10 @@ POSTGRES_USER=podokit
 POSTGRES_PASSWORD=podokit
 POSTGRES_DB=podokit
 BACKEND_INTERNAL_URL=http://api:5002
+# Web: milliseconds per server-side API call, and the /api proxy's wait for
+# response headers.
+# BACKEND_TIMEOUT_MS=2000
+# BACKEND_PROXY_TIMEOUT_MS=30000
 CORS_ORIGIN=http://{{projectName}}.localhost,http://localhost
 REDIS_HOST=redis
 REDIS_PORT=6379

package/dist/templates/fullstack/dot-env.example

@@ -35,6 +35,13 @@ POSTGRES_DB=podokit
 
 # Web -> API boundary (server-side only; not exposed to the browser)
 BACKEND_INTERNAL_URL=http://localhost:5002
+# Milliseconds one server-side API call may take, and how long the /api proxy
+# waits for response headers (504 BACKEND_TIMEOUT after the second value). When
+# the API's GET /health does not answer within BACKEND_TIMEOUT_MS either, API
+# calls fail at once until it does. Raise BACKEND_PROXY_TIMEOUT_MS together with
+# the API's apiServerOptions.idleTimeout.
+# BACKEND_TIMEOUT_MS=2000
+# BACKEND_PROXY_TIMEOUT_MS=30000
 
 CORS_ORIGIN=http://localhost:5001
 # Web request limit; leave room for a 2 MiB multipart upload

package/dist/templates/fullstack/infra/k3s/web-deployment.yaml

@@ -21,10 +21,13 @@ spec:
           envFrom:
             - configMapRef: { name: app-config }
           # Without these a rolling update sends traffic to a process that has not
-          # bound its port yet.
+          # bound its port yet. /healthz never calls the API, so an API outage does
+          # not take the web pods out of service.
           readinessProbe:
-            httpGet: { path: /, port: 3000 }
+            httpGet: { path: /healthz, port: 3000 }
             initialDelaySeconds: 5
+            timeoutSeconds: 2
           livenessProbe:
-            httpGet: { path: /, port: 3000 }
+            httpGet: { path: /healthz, port: 3000 }
             initialDelaySeconds: 15
+            timeoutSeconds: 2

package/dist/templates/fullstack/tests/api/health.api.spec.ts

@@ -12,3 +12,11 @@ test("GET /health/ready reports the database", async ({ request }) => {
   expect(res.status()).toBe(200);
   expect((await res.json()).status).toBe("ready");
 });
+
+test("GET /healthz answers from the web server without caching or indexing @smoke", async ({ request }) => {
+  const res = await request.get("/healthz");
+  expect(res.status()).toBe(200);
+  expect(await res.text()).toBe("ok\n");
+  expect(res.headers()["cache-control"]).toBe("no-store");
+  expect(res.headers()["x-robots-tag"]).toBe("noindex, nofollow");
+});

package/dist/templates/modules/admin-dashboard/files/apps/web/src/hooks.server.test.ts

@@ -1,6 +1,7 @@
 import { ApiError } from "@podosoft/podokit-api-client";
 import type { RequestEvent } from "@sveltejs/kit";
 import { beforeEach, describe, expect, it, vi } from "vitest";
+import { BackendUnavailableError } from "#lib/server/api.js";
 import { handle } from "./hooks.server";
 
 type SessionResult = {
@@ -13,7 +14,8 @@ const client = vi.hoisted(() => ({
   getSession: vi.fn<() => Promise<SessionResult>>(),
 }));
 
-vi.mock("#lib/server/api.js", () => ({
+vi.mock("#lib/server/api.js", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("#lib/server/api.js")>()),
   serverApiClient: () => ({ get: client.get, auth: { getSession: client.getSession } }),
 }));
 
@@ -98,6 +100,16 @@ describe("handle", () => {
     });
   });
 
+  it("treats an API known to be unreachable as an outage", async () => {
+    client.get.mockRejectedValue(new BackendUnavailableError());
+    client.getSession.mockRejectedValue(new BackendUnavailableError());
+
+    await expect(handle({ event: requestEvent("/account"), resolve })).rejects.toMatchObject({ status: 503 });
+    const event = requestEvent("/");
+    await handle({ event, resolve });
+    expect(event.locals).toMatchObject({ site: null, siteUnavailable: true, authUnavailable: true, user: null });
+  });
+
   it("confirms an anonymous visitor on a 401 session response", async () => {
     client.get.mockResolvedValue({ locale: null });
     client.getSession.mockResolvedValue({ data: null, error: { status: 401 } });
@@ -121,4 +133,15 @@ describe("handle", () => {
     expect(client.get).not.toHaveBeenCalled();
     expect(client.getSession).not.toHaveBeenCalled();
   });
+
+  it("answers the health route without runtime lookups", async () => {
+    client.get.mockRejectedValue(new BackendUnavailableError());
+    client.getSession.mockRejectedValue(new BackendUnavailableError());
+
+    await handle({ event: requestEvent("/healthz"), resolve });
+
+    expect(client.get).not.toHaveBeenCalled();
+    expect(client.getSession).not.toHaveBeenCalled();
+    expect(resolve).toHaveBeenCalledTimes(1);
+  });
 });

package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/runtime-context.test.ts

@@ -61,6 +61,12 @@ describe("needsRuntimeContext", () => {
     expect(needsRuntimeContext(request("/apiary", { headers: { accept: NAVIGATION } }))).toBe(true);
   });
 
+  it("skips them for the web server's health route, which must answer while the API is down", () => {
+    expect(needsRuntimeContext(request("/healthz", { headers: { accept: "*/*" } }))).toBe(false);
+    expect(needsRuntimeContext(request("/healthz", { headers: { accept: NAVIGATION } }))).toBe(false);
+    expect(needsRuntimeContext(request("/healthz/extra", { routeId: null, headers: { accept: NAVIGATION } }))).toBe(true);
+  });
+
   it("skips them while prerendering", () => {
     expect(needsRuntimeContext(request("/pricing", { headers: { accept: NAVIGATION } }), { building: true })).toBe(false);
   });

package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/runtime-context.ts

@@ -1,5 +1,8 @@
 import { APP_ASSET_PATHS } from "./asset-paths";
 
+/** The web server's own health route (`routes/healthz/+server.ts`), which never calls the API. */
+const WEB_HEALTH_PATH = "/healthz";
+
 /** The request fields that decide whether a response can use the session and site settings. */
 export interface RuntimeContextRequest {
   url: URL;
@@ -92,6 +95,8 @@ export function needsRuntimeContext(
   const { pathname } = event.url;
   // The backend proxy forwards the cookies; the API resolves the session itself.
   if (underPrefix(pathname, "/api")) return false;
+  // The web server's health route answers the probes even while the API is down.
+  if (pathname === WEB_HEALTH_PATH) return false;
   // Remote functions run application code that may read the session.
   if (event.isRemoteRequest) return true;
   if (event.route.id === null) {