@growthbook/edge-utils 0.1.7 → 0.2.0

package/src/app.ts

@@ -12,6 +12,15 @@ import { applyDomMutations } from "./domMutations";
 import redirect from "./redirect";
 import { getRoute } from "./routing";
 import { EdgeStickyBucketService } from "./stickyBucketService";
+import { HTMLElement, parse } from "node-html-parser";
+import pako from "pako";
+
+interface OriginResponse {
+  status: number;
+  headers: Record<string, string | undefined>;
+  text: () => Promise<string>;
+  arrayBuffer: () => Promise<ArrayBuffer>;
+}
 
 export async function edgeApp<Req, Res>(
   context: Context<Req, Res>,
@@ -20,68 +29,78 @@ export async function edgeApp<Req, Res>(
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   next?: any,
 ) {
-  let url = context.helpers.getRequestURL?.(req) || "";
-
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  let headers: Record<string, any> = {
-    "Content-Type": "text/html",
+  /**
+   * 1. Init app variables
+   */
+    // Request vars:
+  let requestUrl = context.helpers.getRequestURL(req);
+  let originUrl = getOriginUrl(context, requestUrl);
+  // Response vars:
+  let originResponse: (OriginResponse & Res) | undefined = undefined;
+  let resHeaders: Record<string, string | undefined> = {};
+  const respCookies: Record<string, string> = {};
+  const setRespCookie = (key: string, value: string) => {
+    respCookies[key] = value;
   };
-  const cookies: Record<string, string> = {};
-  const setCookie = (key: string, value: string) => {
-    cookies[key] = value;
-  };
-  const { csp, nonce } = getCspInfo(context as Context<unknown, unknown>);
-  if (csp) {
-    headers["Content-Security-Policy"] = csp;
-  }
-  let body = "";
+  // Initial hook:
+  let hookResp: Res | undefined | void;
+  hookResp = await context?.hooks?.onRequest?.({ context, req, res, next, requestUrl, originUrl });
+  if (hookResp) return hookResp;
+
+  // DOM mutations
+  let domChanges: AutoExperimentVariation[] = [];
+  const resetDomChanges = () => (domChanges = []);
+
+  // Experiments that triggered prior to final redirect
+  let preRedirectChangeIds: string[] = [];
+  const setPreRedirectChangeIds = (changeIds: string[]) =>
+    (preRedirectChangeIds = changeIds);
 
+  /**
+   * 2. Early exits based on method, routes, etc
+   */
   // Non GET requests are proxied
-  if (context.helpers.getRequestMethod?.(req) !== "GET") {
-    return context.helpers.proxyRequest?.(context, req, res, next);
+  if (context.helpers.getRequestMethod(req) !== "GET") {
+    return context.helpers.proxyRequest(context, req, res, next);
   }
   // Check the url for routing rules (default behavior is intercept)
-  const route = getRoute(context as Context<unknown, unknown>, url);
+  const route = getRoute(context, requestUrl);
   if (route.behavior === "error") {
-    return context.helpers.sendResponse?.(
-      context,
-      res,
-      headers,
-      route.body || "",
-      cookies,
-      route.statusCode,
-    );
+    return context.helpers.sendResponse(context, res, {}, route.body || "", {}, route.statusCode);
   }
   if (route.behavior === "proxy") {
-    return context.helpers.proxyRequest?.(context, req, res, next);
+    return context.helpers.proxyRequest(context, req, res, next);
   }
+  // Custom route behavior via hook:
+  hookResp = await context?.hooks?.onRoute?.({ context, req, res, next, requestUrl, originUrl, route });
+  if (hookResp) return hookResp;
 
-  const attributes = getUserAttributes(context, req, url, setCookie);
-
-  let domChanges: AutoExperimentVariation[] = [];
-  const resetDomChanges = () => (domChanges = []);
+  /**
+   * 3. User attributes & uuid
+   */
+  const attributes = getUserAttributes(context, req, requestUrl, setRespCookie);
 
-  let preRedirectChangeIds: string[] = [];
-  const setPreRedirectChangeIds = (changeIds: string[]) =>
-    (preRedirectChangeIds = changeIds);
+  // Hook to allow enriching user attributes, etc
+  hookResp = await context?.hooks?.onUserAttributes?.({ context, req, res, next, requestUrl, originUrl, route, attributes });
+  if (hookResp) return hookResp;
 
-  if (context.config.localStorage)
-    setPolyfills({ localStorage: context.config.localStorage });
-  if (context.config.crypto)
-    setPolyfills({ SubtleCrypto: context.config.crypto });
+  /**
+   * 4. Init GrowthBook SDK
+   */
+  setPolyfills({
+    localStorage: context.config?.localStorage,
+    SubtleCrypto: context.config?.crypto,
+  });
   if (context.config.staleTTL !== undefined)
     configureCache({ staleTTL: context.config.staleTTL });
   if (context.config.fetchFeaturesCall)
     helpers.fetchFeaturesCall = context.config.fetchFeaturesCall;
 
-  let stickyBucketService:
-    | EdgeStickyBucketService<Req, Res>
-    | StickyBucketService
-    | undefined = undefined;
+  let stickyBucketService: EdgeStickyBucketService<Req, Res> | StickyBucketService | undefined;
   if (context.config.enableStickyBucketing) {
     stickyBucketService =
       context.config.edgeStickyBucketService ??
-      new EdgeStickyBucketService<Req, Res>({
+      new EdgeStickyBucketService({
         context,
         prefix: context.config.stickyBucketPrefix,
         req,
@@ -94,9 +113,10 @@ export async function edgeApp<Req, Res>(
     attributes,
     applyDomChangesCallback: (changes: AutoExperimentVariation) => {
       domChanges.push(changes);
-      return () => {};
+      return () => {
+      };
     },
-    url,
+    url: requestUrl,
     disableVisualExperiments: ["skip", "browser"].includes(
       context.config.runVisualEditorExperiments,
     ),
@@ -115,81 +135,140 @@ export async function edgeApp<Req, Res>(
     payload: context.config.payload,
   });
 
-  const oldUrl = url;
-  url = await redirect({
-    context: context as Context<unknown, unknown>,
+  // Hook to perform any custom logic given the initialized SDK
+  hookResp = await context?.hooks?.onGrowthbookInit?.({ context, req, res, next, requestUrl, originUrl, route, attributes, growthbook });
+  if (hookResp) return hookResp;
+
+
+  /**
+   * 5. Run URL redirect tests before fetching from origin
+   */
+  const redirectRequestUrl = await redirect({
+    context,
     req,
-    setCookie,
+    setRespCookie,
     growthbook,
-    previousUrl: url,
+    previousUrl: requestUrl,
     resetDomChanges,
     setPreRedirectChangeIds: setPreRedirectChangeIds,
   });
+  originUrl = getOriginUrl(context, redirectRequestUrl);
 
-  const originUrl = getOriginUrl(context as Context<unknown, unknown>, url);
+  // Pre-origin-fetch hook (after redirect logic):
+  hookResp = await context?.hooks?.onBeforeOriginFetch?.({ context, req, res, next, requestUrl, redirectRequestUrl, originUrl, route, attributes, growthbook });
+  if (hookResp) return hookResp;
 
-  /* eslint-disable @typescript-eslint/no-explicit-any */
-  let fetchedResponse:
-    | (Res & { status: number; headers: Record<string, any>; text: any })
-    | undefined = undefined;
+  /**
+   * 6. Fetch from origin, parse body / DOM
+   */
   try {
-    fetchedResponse = (await context.helpers.fetch?.(
-      context as Context<Req, Res>,
+    originResponse = await context.helpers.fetch(
+      context,
       originUrl,
-      /* eslint-disable @typescript-eslint/no-explicit-any */
-    )) as Res & { status: number; headers: Record<string, any>; text: any };
-    const status = parseInt(fetchedResponse.status ? fetchedResponse.status + "" : "400");
-    if (status >= 500) {
-      console.error("Fetch: 5xx status returned");
-      return context.helpers.sendResponse?.(
-        context,
-        res,
-        headers,
-        "Error fetching page",
-        cookies,
-        500,
-      );
-    }
-    if (status >= 400) {
-      return context.helpers.proxyRequest?.(context, req, res, next);
-    }
+      req,
+    ) as OriginResponse & Res;
   } catch (e) {
     console.error(e);
-    return context.helpers.sendResponse?.(
-      context,
-      res,
-      headers,
-      "Error fetching page",
-      cookies,
-      500,
-    );
   }
-  if (context.config.forwardProxyHeaders && fetchedResponse?.headers) {
-    headers = { ...fetchedResponse.headers, ...headers };
+  const originStatus = originResponse ? parseInt(originResponse.status ? originResponse.status + "" : "400") : 500;
+
+  // On fetch hook (for custom response processing, etc)
+  hookResp = await context?.hooks?.onOriginFetch?.({ context, req, res, next, requestUrl, redirectRequestUrl, originUrl, route, attributes, growthbook, originResponse, originStatus });
+  if (hookResp) return hookResp;
+
+  // Standard error response handling
+  if (originStatus >= 500 || !originResponse) {
+    console.error("Fetch: 5xx status returned");
+    return context.helpers.sendResponse(context, res, {}, "Error fetching page", {}, 500);
+  }
+  if (originStatus >= 400) {
+    return originResponse;
+  }
+
+  // Got a valid response, begin processing
+  const originHeaders = headersToObject(originResponse.headers);
+  if (context.config.forwardProxyHeaders) {
+    resHeaders = { ...originHeaders, ...resHeaders };
+  }
+  // At minimum, the content-type is forwarded
+  resHeaders["content-type"] = originHeaders?.["content-type"];
+
+  if (context.config.useDefaultContentType && !resHeaders["content-type"]) {
+    resHeaders["content-type"] = "text/html";
+  }
+  if (context.config.processTextHtmlOnly && !(resHeaders["content-type"] ?? "").includes("text/html")) {
+    return context.helpers.proxyRequest(context, req, res, next);
   }
-  body = await fetchedResponse.text();
 
-  body = await applyDomMutations({
+  const { csp, nonce } = getCspInfo(context);
+  if (csp) {
+    resHeaders["content-security-policy"] = csp;
+  }
+
+  let body: string = "";
+  try {
+    // Check if content-encoding is gzip
+    if (originHeaders["content-encoding"] === "gzip") {
+      const buffer = await originResponse.arrayBuffer();
+      body = pako.inflate(new Uint8Array(buffer), { to: "string" });
+      delete resHeaders["content-encoding"]; // do not forward this header since it's now unzipped
+    } else {
+      body = await originResponse?.text() ?? "";
+    }
+  } catch(e) {
+    console.error(e);
+  }
+  let setBody = (s: string) => {
+    body = s;
+  }
+
+  let root: HTMLElement | undefined;
+  if (context.config.alwaysParseDOM) {
+    root = parse(body);
+  }
+
+  // Body ready hook (pre-DOM-mutations):
+  hookResp = await context?.hooks?.onBodyReady?.({ context, req, res, next, requestUrl, redirectRequestUrl, originUrl, route, attributes, growthbook, originResponse, originStatus, originHeaders, resHeaders, body, setBody, root });
+  if (hookResp) return hookResp;
+
+  /**
+   * 7. Apply visual editor DOM mutations
+   */
+  await applyDomMutations({
     body,
+    setBody,
+    root,
     nonce,
     domChanges,
   });
 
-  body = injectScript({
-    context: context as Context<unknown, unknown>,
+  /**
+   * 8. Inject the client-facing GrowthBook SDK (auto-wrapper)
+   */
+  injectScript({
+    context,
     body,
+    setBody,
     nonce,
     growthbook,
     attributes,
     preRedirectChangeIds,
-    url,
-    oldUrl,
+    url: redirectRequestUrl,
+    oldUrl: requestUrl,
   });
 
-  return context.helpers.sendResponse?.(context, res, headers, body, cookies);
+  // Final hook (post-mutations) before sending back
+  hookResp = await context?.hooks?.onBeforeResponse?.({ context, req, res, next, requestUrl, redirectRequestUrl, originUrl, route, attributes, growthbook, originResponse, originStatus, originHeaders, resHeaders, body, setBody });
+  if (hookResp) return hookResp;
+
+  /**
+   * 9. Send mutated response
+   */
+  return context.helpers.sendResponse(context, res, resHeaders, body, respCookies);
 }
 
-export function getOriginUrl(context: Context, currentURL: string): string {
+
+export function getOriginUrl<Req, Res>(context: Context<Req, Res>, currentURL: string): string {
   const proxyTarget = context.config.proxyTarget;
   const currentParsedURL = new URL(currentURL);
   const proxyParsedURL = new URL(proxyTarget);
@@ -220,3 +299,10 @@ export function getOriginUrl(context: Context, currentURL: string): string {
 
   return newURL;
 }
+
+function headersToObject(headers: any) {
+  if (headers && typeof headers.entries === "function") {
+    return Object.fromEntries(headers.entries());
+  }
+  return headers || {};
+}

package/src/attributes.ts

@@ -6,7 +6,7 @@ export function getUserAttributes<Req, Res>(
   ctx: Context<Req, Res>,
   req: Req,
   url: string,
-  setCookie: (key: string, value: string) => void,
+  setRespCookie: (key: string, value: string) => void,
 ): Attributes {
   const { config, helpers } = ctx;
 
@@ -20,7 +20,7 @@ export function getUserAttributes<Req, Res>(
     if (!helpers?.setCookie) {
       throw new Error("Missing required dependencies");
     }
-    setCookie(config.uuidCookieName, uuid);
+    setRespCookie(config.uuidCookieName, uuid);
   }
 
   const autoAttributes = getAutoAttributes(ctx, req, url);
@@ -34,7 +34,6 @@ export function getUUID<Req, Res>(ctx: Context<Req, Res>, req: Req) {
   const { config, helpers } = ctx;
 
   const crypto = config?.crypto || globalThis?.crypto;
-
   if (!crypto || !helpers?.getCookie) {
     throw new Error("Missing required dependencies");
   }

package/src/config.ts

@@ -1,13 +1,20 @@
 import { Config, Context, ExperimentRunEnvironment } from "./types";
 
-export const defaultContext: Context = {
+type Req = any; // placeholder
+type Res = any; // placeholder
+
+export const defaultContext: Context<Req, Res> = {
   config: {
     proxyTarget: "/",
     forwardProxyHeaders: true,
+    followRedirects: true,
+    useDefaultContentType: false,
+    processTextHtmlOnly: true,
     environment: "production",
     maxPayloadSize: "2mb",
     runVisualEditorExperiments: "everywhere",
     disableJsInjection: false,
+    alwaysParseDOM: false,
     runUrlRedirectExperiments: "browser",
     runCrossOriginUrlRedirectExperiments: "browser",
     injectRedirectUrlScript: true,
@@ -24,12 +31,32 @@ export const defaultContext: Context = {
     uuidKey: "id",
     skipAutoAttributes: false,
   },
-  helpers: {},
+  helpers: {
+    getRequestURL: function(req: Req): string {
+      throw new Error("getRequestURL not implemented");
+    },
+    getRequestMethod: function(req: Req): string {
+      throw new Error("getRequestMethod not implemented");
+    },
+    sendResponse: function(ctx: Context<Req, Res>, res?: any, headers?: Record<string, any> | undefined, body?: string | undefined, cookies?: Record<string, string> | undefined, status?: number | undefined): unknown {
+      throw new Error("sendResponse not implemented");
+    },
+    fetch: function(ctx: Context<Req, Res>, url: string, req: Req): Promise<Res> {
+      throw new Error("fetchFn not implemented");
+    },
+    proxyRequest: function(ctx: Context<Req, Res>, req: Req, res?: any, next?: any): Promise<unknown> {
+      throw new Error("proxyRequest not implemented");
+    }
+  },
+  hooks: {},
 };
 
 export interface ConfigEnv {
   PROXY_TARGET?: string;
   FORWARD_PROXY_HEADERS?: string;
+  FOLLOW_REDIRECTS?: string;
+  USE_DEFAULT_CONTENT_TYPE?: string;
+  PROCESS_TEXT_HTML_ONLY?: string;
   NODE_ENV?: string;
   MAX_PAYLOAD_SIZE?: string;
 
@@ -37,6 +64,7 @@ export interface ConfigEnv {
 
   RUN_VISUAL_EDITOR_EXPERIMENTS?: ExperimentRunEnvironment;
   DISABLE_JS_INJECTION?: string;
+  ALWAYS_PARSE_DOM?: string;
 
   RUN_URL_REDIRECT_EXPERIMENTS?: ExperimentRunEnvironment;
   RUN_CROSS_ORIGIN_URL_REDIRECT_EXPERIMENTS?: ExperimentRunEnvironment;
@@ -79,6 +107,15 @@ export function getConfig(env: ConfigEnv): Config {
   config.forwardProxyHeaders = ["true", "1"].includes(
     env.FORWARD_PROXY_HEADERS ?? "" + defaultContext.config.forwardProxyHeaders,
   );
+  config.followRedirects = ["true", "1"].includes(
+    env.FOLLOW_REDIRECTS ?? "" + defaultContext.config.followRedirects,
+  );
+  config.useDefaultContentType = ["true", "1"].includes(
+    env.USE_DEFAULT_CONTENT_TYPE ?? "" + defaultContext.config.useDefaultContentType,
+  );
+  config.processTextHtmlOnly = ["true", "1"].includes(
+    env.PROCESS_TEXT_HTML_ONLY ?? "" + defaultContext.config.processTextHtmlOnly,
+  );
   config.environment = env.NODE_ENV ?? defaultContext.config.environment;
   config.maxPayloadSize =
     env.MAX_PAYLOAD_SIZE ?? defaultContext.config.maxPayloadSize;
@@ -96,6 +133,9 @@ export function getConfig(env: ConfigEnv): Config {
   config.disableJsInjection = ["true", "1"].includes(
     env.DISABLE_JS_INJECTION ?? "" + defaultContext.config.disableJsInjection,
   );
+  config.alwaysParseDOM = ["true", "1"].includes(
+    env.ALWAYS_PARSE_DOM ?? "" + defaultContext.config.alwaysParseDOM,
+  );
 
   config.runUrlRedirectExperiments = (env.RUN_URL_REDIRECT_EXPERIMENTS ??
     defaultContext.config

package/src/domMutations.ts

@@ -1,23 +1,29 @@
 import { AutoExperimentVariation, DOMMutation } from "@growthbook/growthbook";
-import { parse } from "node-html-parser";
+import { HTMLElement, parse } from "node-html-parser";
 
 export async function applyDomMutations({
   body,
+  setBody,
+  root,
   nonce,
   domChanges,
 }: {
   body: string;
+  setBody: (s: string) => void;
+  root?: HTMLElement;
   nonce?: string;
   domChanges: AutoExperimentVariation[];
 }) {
-  if (!domChanges.length) return body;
+  if (!domChanges.length) return;
+  root = root ?? parse(body);
+  if (!root) return;
 
-  const root = parse(body);
   const headEl = root.querySelector("head");
 
   domChanges.forEach(({ domMutations, css, js }) => {
     if (css) {
       const parentEl = headEl || root;
+      if (!parentEl) return;
       const el = parse(`<style>${css}</style>`);
       parentEl.appendChild(el);
     }
@@ -86,9 +92,11 @@ export async function applyDomMutations({
   });
 
   body = root.toString();
-  return body;
+  setBody(body);
+  return;
 
   function html(selector: string, cb: (val: string) => string) {
+    if (!root) return;
     const els = root.querySelectorAll(selector);
     els.map((el) => {
       el.innerHTML = cb(el.innerHTML);
@@ -96,6 +104,7 @@ export async function applyDomMutations({
   }
 
   function classes(selector: string, cb: (val: Set<string>) => void) {
+    if (!root) return;
     const els = root.querySelectorAll(selector);
     els.map((el) => {
       const classList = new Set(el.classNames);
@@ -109,6 +118,7 @@ export async function applyDomMutations({
     attr: string,
     cb: (val: string | null) => string | null,
   ) {
+    if (!root) return;
     const validAttributeName = /^[a-zA-Z:_][a-zA-Z0-9:_.-]*$/;
     if (!validAttributeName.test(attr)) {
       return;
@@ -139,8 +149,10 @@ export async function applyDomMutations({
     selector: string,
     cb: () => { insertBeforeSelector?: string; parentSelector: string },
   ) {
+    if (!root) return;
     const els = root.querySelectorAll(selector);
     els.map((el) => {
+      if (!root) return;
       const { insertBeforeSelector, parentSelector } = cb();
       const parent = root.querySelector(parentSelector);
       const insertBefore = insertBeforeSelector