@serwist/sw 9.0.0-preview.17 → 9.0.0-preview.18

package/src/plugins/expiration/ExpirationPlugin.ts

@@ -0,0 +1,297 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { CacheDidUpdateCallbackParam, CachedResponseWillBeUsedCallbackParam, SerwistPlugin } from "@serwist/core";
+import { registerQuotaErrorCallback } from "@serwist/core";
+import { assert, SerwistError, getFriendlyURL, logger, privateCacheNames } from "@serwist/core/internal";
+
+import { CacheExpiration } from "./CacheExpiration.js";
+
+export interface ExpirationPluginOptions {
+  /**
+   * The maximum number of entries to cache. Entries used the least will be removed
+   * as the maximum is reached.
+   */
+  maxEntries?: number;
+  /**
+   * The maximum number of seconds before an entry is treated as stale and removed.
+   */
+  maxAgeSeconds?: number;
+  /**
+   * Determines whether `maxAgeSeconds` should be calculated from when an
+   * entry was last fetched or when it was last used.
+   *
+   * @default "last-fetched"
+   */
+  maxAgeFrom?: "last-fetched" | "last-used";
+  /**
+   * The [`CacheQueryOptions`](https://developer.mozilla.org/en-US/docs/Web/API/Cache/delete#Parameters)
+   * that will be used when calling `delete()` on the cache.
+   */
+  matchOptions?: CacheQueryOptions;
+  /**
+   * Whether to opt this cache into automatic deletion if the available storage quota has been exceeded.
+   */
+  purgeOnQuotaError?: boolean;
+}
+
+/**
+ * This plugin can be used in a `@serwist/strategies` Strategy to regularly enforce a
+ * limit on the age and/or the number of cached requests.
+ *
+ * It can only be used with Strategy instances that have a custom `cacheName` property set.
+ * In other words, it can't be used to expire entries in strategies that use the default runtime
+ * cache name.
+ *
+ * Whenever a cached response is used or updated, this plugin will look
+ * at the associated cache and remove any old or extra responses.
+ *
+ * When using `maxAgeSeconds`, responses may be used *once* after expiring
+ * because the expiration clean up will not have occurred until *after* the
+ * cached response has been used. If the response has a "Date" header, then a lightweight expiration
+ * check is performed, and the response will not be used immediately.
+ *
+ * When using `maxEntries`, the least recently requested entry will be removed
+ * from the cache.
+ *
+ * @see https://serwist.pages.dev/docs/expiration/expiration-plugin
+ */
+export class ExpirationPlugin implements SerwistPlugin {
+  private readonly _config: ExpirationPluginOptions;
+  private _cacheExpirations: Map<string, CacheExpiration>;
+
+  /**
+   * @param config
+   */
+  constructor(config: ExpirationPluginOptions = {}) {
+    if (process.env.NODE_ENV !== "production") {
+      if (!(config.maxEntries || config.maxAgeSeconds)) {
+        throw new SerwistError("max-entries-or-age-required", {
+          moduleName: "@serwist/expiration",
+          className: "ExpirationPlugin",
+          funcName: "constructor",
+        });
+      }
+
+      if (config.maxEntries) {
+        assert!.isType(config.maxEntries, "number", {
+          moduleName: "@serwist/expiration",
+          className: "ExpirationPlugin",
+          funcName: "constructor",
+          paramName: "config.maxEntries",
+        });
+      }
+
+      if (config.maxAgeSeconds) {
+        assert!.isType(config.maxAgeSeconds, "number", {
+          moduleName: "@serwist/expiration",
+          className: "ExpirationPlugin",
+          funcName: "constructor",
+          paramName: "config.maxAgeSeconds",
+        });
+      }
+
+      if (config.maxAgeFrom) {
+        assert!.isType(config.maxAgeFrom, "string", {
+          moduleName: "@serwist/expiration",
+          className: "ExpirationPlugin",
+          funcName: "constructor",
+          paramName: "config.maxAgeFrom",
+        });
+      }
+    }
+
+    this._config = config;
+    this._cacheExpirations = new Map();
+
+    if (!this._config.maxAgeFrom) {
+      this._config.maxAgeFrom = "last-fetched";
+    }
+
+    if (this._config.purgeOnQuotaError) {
+      registerQuotaErrorCallback(() => this.deleteCacheAndMetadata());
+    }
+  }
+
+  /**
+   * A simple helper method to return a CacheExpiration instance for a given
+   * cache name.
+   *
+   * @param cacheName
+   * @returns
+   * @private
+   */
+  private _getCacheExpiration(cacheName: string): CacheExpiration {
+    if (cacheName === privateCacheNames.getRuntimeName()) {
+      throw new SerwistError("expire-custom-caches-only");
+    }
+
+    let cacheExpiration = this._cacheExpirations.get(cacheName);
+    if (!cacheExpiration) {
+      cacheExpiration = new CacheExpiration(cacheName, this._config);
+      this._cacheExpirations.set(cacheName, cacheExpiration);
+    }
+    return cacheExpiration;
+  }
+
+  /**
+   * A "lifecycle" callback that will be triggered automatically by the
+   * `@serwist/strategies` handlers when a `Response` is about to be returned
+   * from a [Cache](https://developer.mozilla.org/en-US/docs/Web/API/Cache) to
+   * the handler. It allows the `Response` to be inspected for freshness and
+   * prevents it from being used if the `Response`'s `Date` header value is
+   * older than the configured `maxAgeSeconds`.
+   *
+   * @param options
+   * @returns `cachedResponse` if it is fresh and `null` if it is stale or
+   * not available.
+   * @private
+   */
+  cachedResponseWillBeUsed({ event, cacheName, request, cachedResponse }: CachedResponseWillBeUsedCallbackParam) {
+    if (!cachedResponse) {
+      return null;
+    }
+
+    const isFresh = this._isResponseDateFresh(cachedResponse);
+
+    // Expire entries to ensure that even if the expiration date has
+    // expired, it'll only be used once.
+    const cacheExpiration = this._getCacheExpiration(cacheName);
+
+    const isMaxAgeFromLastUsed = this._config.maxAgeFrom === "last-used";
+
+    const done = (async () => {
+      // Update the metadata for the request URL to the current timestamp.
+      // Only applies if `maxAgeFrom` is `"last-used"`, since the current
+      // lifecycle callback is `cachedResponseWillBeUsed`.
+      // This needs to be called before `expireEntries()` so as to avoid
+      // this URL being marked as expired.
+      if (isMaxAgeFromLastUsed) {
+        await cacheExpiration.updateTimestamp(request.url);
+      }
+      await cacheExpiration.expireEntries();
+    })();
+    try {
+      event.waitUntil(done);
+    } catch (error) {
+      if (process.env.NODE_ENV !== "production") {
+        if (event instanceof FetchEvent) {
+          logger.warn(`Unable to ensure service worker stays alive when updating cache entry for '${getFriendlyURL(event.request.url)}'.`);
+        }
+      }
+    }
+
+    return isFresh ? cachedResponse : null;
+  }
+
+  /**
+   * @param cachedResponse
+   * @returns
+   * @private
+   */
+  private _isResponseDateFresh(cachedResponse: Response): boolean {
+    const isMaxAgeFromLastUsed = this._config.maxAgeFrom === "last-used";
+    // If `maxAgeFrom` is `"last-used"`, the `Date` header doesn't really
+    // matter since it is about when the response was created.
+    if (isMaxAgeFromLastUsed) {
+      return true;
+    }
+    const now = Date.now();
+    if (!this._config.maxAgeSeconds) {
+      return true;
+    }
+    // Check if the `Date` header will suffice a quick expiration check.
+    // See https://github.com/GoogleChromeLabs/sw-toolbox/issues/164 for
+    // discussion.
+    const dateHeaderTimestamp = this._getDateHeaderTimestamp(cachedResponse);
+    if (dateHeaderTimestamp === null) {
+      // Unable to parse date, so assume it's fresh.
+      return true;
+    }
+    // If we have a valid headerTime, then our response is fresh if the
+    // headerTime plus maxAgeSeconds is greater than the current time.
+    return dateHeaderTimestamp >= now - this._config.maxAgeSeconds * 1000;
+  }
+
+  /**
+   * Extracts the `Date` header and parse it into an useful value.
+   *
+   * @param cachedResponse
+   * @returns
+   * @private
+   */
+  private _getDateHeaderTimestamp(cachedResponse: Response): number | null {
+    if (!cachedResponse.headers.has("date")) {
+      return null;
+    }
+
+    const dateHeader = cachedResponse.headers.get("date")!;
+    const parsedDate = new Date(dateHeader);
+    const headerTime = parsedDate.getTime();
+
+    // If the `Date` header is invalid for some reason, `parsedDate.getTime()`
+    // will return NaN.
+    if (Number.isNaN(headerTime)) {
+      return null;
+    }
+
+    return headerTime;
+  }
+
+  /**
+   * A "lifecycle" callback that will be triggered automatically by the
+   * `@serwist/strategies` handlers when an entry is added to a cache.
+   *
+   * @param options
+   * @private
+   */
+  async cacheDidUpdate({ cacheName, request }: CacheDidUpdateCallbackParam) {
+    if (process.env.NODE_ENV !== "production") {
+      assert!.isType(cacheName, "string", {
+        moduleName: "@serwist/expiration",
+        className: "Plugin",
+        funcName: "cacheDidUpdate",
+        paramName: "cacheName",
+      });
+      assert!.isInstance(request, Request, {
+        moduleName: "@serwist/expiration",
+        className: "Plugin",
+        funcName: "cacheDidUpdate",
+        paramName: "request",
+      });
+    }
+
+    const cacheExpiration = this._getCacheExpiration(cacheName);
+    await cacheExpiration.updateTimestamp(request.url);
+    await cacheExpiration.expireEntries();
+  }
+
+  /**
+   * Deletes the underlying `Cache` instance associated with this instance and the metadata
+   * from IndexedDB used to keep track of expiration details for each `Cache` instance.
+   *
+   * When using cache expiration, calling this method is preferable to calling
+   * `caches.delete()` directly, since this will ensure that the IndexedDB
+   * metadata is also cleanly removed and that open IndexedDB instances are deleted.
+   *
+   * Note that if you're *not* using cache expiration for a given cache, calling
+   * `caches.delete()` and passing in the cache's name should be sufficient.
+   * There is no Serwist-specific method needed for cleanup in that case.
+   */
+  async deleteCacheAndMetadata(): Promise<void> {
+    // Do this one at a time instead of all at once via `Promise.all()` to
+    // reduce the chance of inconsistency if a promise rejects.
+    for (const [cacheName, cacheExpiration] of this._cacheExpirations) {
+      await self.caches.delete(cacheName);
+      await cacheExpiration.delete();
+    }
+
+    // Reset this._cacheExpirations to its initial state.
+    this._cacheExpirations = new Map();
+  }
+}

package/src/plugins/expiration/models/CacheTimestampsModel.ts

@@ -0,0 +1,184 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { DBSchema, IDBPDatabase } from "idb";
+import { deleteDB, openDB } from "idb";
+
+const DB_NAME = "serwist-expiration";
+const CACHE_OBJECT_STORE = "cache-entries";
+
+const normalizeURL = (unNormalizedUrl: string) => {
+  const url = new URL(unNormalizedUrl, location.href);
+  url.hash = "";
+
+  return url.href;
+};
+
+interface CacheTimestampsModelEntry {
+  id: string;
+  cacheName: string;
+  url: string;
+  timestamp: number;
+}
+
+interface CacheDbSchema extends DBSchema {
+  "cache-entries": {
+    key: string;
+    value: CacheTimestampsModelEntry;
+    indexes: { cacheName: string; timestamp: number };
+  };
+}
+
+/**
+ * Returns the timestamp model.
+ *
+ * @private
+ */
+export class CacheTimestampsModel {
+  private readonly _cacheName: string;
+  private _db: IDBPDatabase<CacheDbSchema> | null = null;
+
+  /**
+   *
+   * @param cacheName
+   *
+   * @private
+   */
+  constructor(cacheName: string) {
+    this._cacheName = cacheName;
+  }
+
+  /**
+   * Takes a URL and returns an ID that will be unique in the object store.
+   *
+   * @param url
+   * @returns
+   * @private
+   */
+  private _getId(url: string): string {
+    return `${this._cacheName}|${normalizeURL(url)}`;
+  }
+
+  /**
+   * Performs an upgrade of indexedDB.
+   *
+   * @param db
+   *
+   * @private
+   */
+  private _upgradeDb(db: IDBPDatabase<CacheDbSchema>) {
+    const objStore = db.createObjectStore(CACHE_OBJECT_STORE, {
+      keyPath: "id",
+    });
+
+    // TODO(philipwalton): once we don't have to support EdgeHTML, we can
+    // create a single index with the keyPath `['cacheName', 'timestamp']`
+    // instead of doing both these indexes.
+    objStore.createIndex("cacheName", "cacheName", { unique: false });
+    objStore.createIndex("timestamp", "timestamp", { unique: false });
+  }
+
+  /**
+   * Performs an upgrade of indexedDB and deletes deprecated DBs.
+   *
+   * @param db
+   *
+   * @private
+   */
+  private _upgradeDbAndDeleteOldDbs(db: IDBPDatabase<CacheDbSchema>) {
+    this._upgradeDb(db);
+    if (this._cacheName) {
+      void deleteDB(this._cacheName);
+    }
+  }
+
+  /**
+   * @param url
+   * @param timestamp
+   *
+   * @private
+   */
+  async setTimestamp(url: string, timestamp: number): Promise<void> {
+    url = normalizeURL(url);
+
+    const entry = {
+      id: this._getId(url),
+      cacheName: this._cacheName,
+      url,
+      timestamp,
+    } satisfies CacheTimestampsModelEntry;
+    const db = await this.getDb();
+    const tx = db.transaction(CACHE_OBJECT_STORE, "readwrite", {
+      durability: "relaxed",
+    });
+    await tx.store.put(entry);
+    await tx.done;
+  }
+
+  /**
+   * Returns the timestamp stored for a given URL.
+   *
+   * @param url
+   * @returns
+   * @private
+   */
+  async getTimestamp(url: string): Promise<number | undefined> {
+    const db = await this.getDb();
+    const entry = await db.get(CACHE_OBJECT_STORE, this._getId(url));
+    return entry?.timestamp;
+  }
+
+  /**
+   * Iterates through all the entries in the object store (from newest to
+   * oldest) and removes entries once either `maxCount` is reached or the
+   * entry's timestamp is less than `minTimestamp`.
+   *
+   * @param minTimestamp
+   * @param maxCount
+   * @returns
+   * @private
+   */
+  async expireEntries(minTimestamp: number, maxCount?: number): Promise<string[]> {
+    const db = await this.getDb();
+    let cursor = await db.transaction(CACHE_OBJECT_STORE, "readwrite").store.index("timestamp").openCursor(null, "prev");
+    const urlsDeleted: string[] = [];
+    let entriesNotDeletedCount = 0;
+    while (cursor) {
+      const result = cursor.value;
+      // TODO(philipwalton): once we can use a multi-key index, we
+      // won't have to check `cacheName` here.
+      if (result.cacheName === this._cacheName) {
+        // Delete an entry if it's older than the max age or
+        // if we already have the max number allowed.
+        if ((minTimestamp && result.timestamp < minTimestamp) || (maxCount && entriesNotDeletedCount >= maxCount)) {
+          cursor.delete();
+          urlsDeleted.push(result.url);
+        } else {
+          entriesNotDeletedCount++;
+        }
+      }
+      cursor = await cursor.continue();
+    }
+
+    return urlsDeleted;
+  }
+
+  /**
+   * Returns an open connection to the database.
+   *
+   * @private
+   */
+  private async getDb() {
+    if (!this._db) {
+      this._db = await openDB(DB_NAME, 1, {
+        upgrade: this._upgradeDbAndDeleteOldDbs.bind(this),
+      });
+    }
+    return this._db;
+  }
+}

package/src/plugins/googleAnalytics/constants.ts

@@ -0,0 +1,22 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+export const QUEUE_NAME = "serwist-google-analytics";
+export const MAX_RETENTION_TIME = 60 * 48; // Two days in minutes
+export const GOOGLE_ANALYTICS_HOST = "www.google-analytics.com";
+export const GTM_HOST = "www.googletagmanager.com";
+export const ANALYTICS_JS_PATH = "/analytics.js";
+export const GTAG_JS_PATH = "/gtag/js";
+export const GTM_JS_PATH = "/gtm.js";
+export const COLLECT_DEFAULT_PATH = "/collect";
+
+// This RegExp matches all known Measurement Protocol single-hit collect
+// endpoints. Most of the time the default path (/collect) is used, but
+// occasionally an experimental endpoint is used when testing new features,
+// (e.g. /r/collect or /j/collect)
+export const COLLECT_PATHS_REGEX = /^\/(\w+\/)?collect/;

package/src/plugins/googleAnalytics/initialize.ts

@@ -0,0 +1,200 @@
+/*
+  Copyright 2018 Google LLC
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+*/
+
+import type { RouteMatchCallbackOptions } from "@serwist/core";
+import { getFriendlyURL, logger, privateCacheNames } from "@serwist/core/internal";
+import { Route } from "../../routing/Route.js";
+import { registerRoute } from "../../routing/registerRoute.js";
+import { NetworkFirst } from "../../strategies/NetworkFirst.js";
+import { NetworkOnly } from "../../strategies/NetworkOnly.js";
+import { BackgroundSyncPlugin } from "../backgroundSync/BackgroundSyncPlugin.js";
+import type { Queue, QueueEntry } from "../backgroundSync/Queue.js";
+
+import {
+  ANALYTICS_JS_PATH,
+  COLLECT_PATHS_REGEX,
+  GOOGLE_ANALYTICS_HOST,
+  GTAG_JS_PATH,
+  GTM_HOST,
+  GTM_JS_PATH,
+  MAX_RETENTION_TIME,
+  QUEUE_NAME,
+} from "./constants.js";
+
+export interface GoogleAnalyticsInitializeOptions {
+  /**
+   * The cache name to store and retrieve analytics.js. Defaults to the cache names provided by `@serwist/core`.
+   */
+  cacheName?: string;
+  /**
+   * [Measurement Protocol parameters](https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters),
+   * expressed as key/value pairs, to be added to replayed Google Analytics
+   * requests. This can be used to, e.g., set a custom dimension indicating
+   * that the request was replayed.
+   */
+  parameterOverrides?: { [paramName: string]: string };
+  /**
+   * A function that allows you to modify the hit parameters prior to replaying
+   * the hit. The function is invoked with the original hit's URLSearchParams
+   * object as its only argument.
+   */
+  hitFilter?: (params: URLSearchParams) => void;
+}
+
+/**
+ * Creates the requestWillDequeue callback to be used with the background
+ * sync plugin. The callback takes the failed request and adds the
+ * `qt` param based on the current time, as well as applies any other
+ * user-defined hit modifications.
+ *
+ * @param config
+ * @returns The requestWillDequeue callback function.
+ * @private
+ */
+const createOnSyncCallback = (config: GoogleAnalyticsInitializeOptions) => {
+  return async ({ queue }: { queue: Queue }) => {
+    let entry: QueueEntry | undefined = undefined;
+    while ((entry = await queue.shiftRequest())) {
+      const { request, timestamp } = entry;
+      const url = new URL(request.url);
+
+      try {
+        // Measurement protocol requests can set their payload parameters in
+        // either the URL query string (for GET requests) or the POST body.
+        const params = request.method === "POST" ? new URLSearchParams(await request.clone().text()) : url.searchParams;
+
+        // Calculate the qt param, accounting for the fact that an existing
+        // qt param may be present and should be updated rather than replaced.
+        const originalHitTime = timestamp! - (Number(params.get("qt")) || 0);
+        const queueTime = Date.now() - originalHitTime;
+
+        // Set the qt param prior to applying hitFilter or parameterOverrides.
+        params.set("qt", String(queueTime));
+
+        // Apply `parameterOverrides`, if set.
+        if (config.parameterOverrides) {
+          for (const param of Object.keys(config.parameterOverrides)) {
+            const value = config.parameterOverrides[param];
+            params.set(param, value);
+          }
+        }
+
+        // Apply `hitFilter`, if set.
+        if (typeof config.hitFilter === "function") {
+          config.hitFilter.call(null, params);
+        }
+
+        // Retry the fetch. Ignore URL search params from the URL as they're
+        // now in the post body.
+        await fetch(
+          new Request(url.origin + url.pathname, {
+            body: params.toString(),
+            method: "POST",
+            mode: "cors",
+            credentials: "omit",
+            headers: { "Content-Type": "text/plain" },
+          }),
+        );
+
+        if (process.env.NODE_ENV !== "production") {
+          logger.log(`Request for '${getFriendlyURL(url.href)}' has been replayed`);
+        }
+      } catch (err) {
+        await queue.unshiftRequest(entry);
+
+        if (process.env.NODE_ENV !== "production") {
+          logger.log(`Request for '${getFriendlyURL(url.href)}' failed to replay, putting it back in the queue.`);
+        }
+        throw err;
+      }
+    }
+    if (process.env.NODE_ENV !== "production") {
+      logger.log("All Google Analytics request successfully replayed; " + "the queue is now empty!");
+    }
+  };
+};
+
+/**
+ * Creates GET and POST routes to catch failed Measurement Protocol hits.
+ *
+ * @param bgSyncPlugin
+ * @returns The created routes.
+ * @private
+ */
+const createCollectRoutes = (bgSyncPlugin: BackgroundSyncPlugin) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GOOGLE_ANALYTICS_HOST && COLLECT_PATHS_REGEX.test(url.pathname);
+
+  const handler = new NetworkOnly({
+    plugins: [bgSyncPlugin],
+  });
+
+  return [new Route(match, handler, "GET"), new Route(match, handler, "POST")];
+};
+
+/**
+ * Creates a route with a network first strategy for the analytics.js script.
+ *
+ * @param cacheName
+ * @returns The created route.
+ * @private
+ */
+const createAnalyticsJsRoute = (cacheName: string) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GOOGLE_ANALYTICS_HOST && url.pathname === ANALYTICS_JS_PATH;
+
+  const handler = new NetworkFirst({ cacheName });
+
+  return new Route(match, handler, "GET");
+};
+
+/**
+ * Creates a route with a network first strategy for the gtag.js script.
+ *
+ * @param cacheName
+ * @returns The created route.
+ * @private
+ */
+const createGtagJsRoute = (cacheName: string) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GTM_HOST && url.pathname === GTAG_JS_PATH;
+
+  const handler = new NetworkFirst({ cacheName });
+
+  return new Route(match, handler, "GET");
+};
+
+/**
+ * Creates a route with a network first strategy for the gtm.js script.
+ *
+ * @param cacheName
+ * @returns The created route.
+ * @private
+ */
+const createGtmJsRoute = (cacheName: string) => {
+  const match = ({ url }: RouteMatchCallbackOptions) => url.hostname === GTM_HOST && url.pathname === GTM_JS_PATH;
+
+  const handler = new NetworkFirst({ cacheName });
+
+  return new Route(match, handler, "GET");
+};
+
+/**
+ * @param options
+ */
+export const initialize = (options: GoogleAnalyticsInitializeOptions = {}): void => {
+  const cacheName = privateCacheNames.getGoogleAnalyticsName(options.cacheName);
+
+  const bgSyncPlugin = new BackgroundSyncPlugin(QUEUE_NAME, {
+    maxRetentionTime: MAX_RETENTION_TIME,
+    onSync: createOnSyncCallback(options),
+  });
+
+  const routes = [createGtmJsRoute(cacheName), createAnalyticsJsRoute(cacheName), createGtagJsRoute(cacheName), ...createCollectRoutes(bgSyncPlugin)];
+
+  for (const route of routes) {
+    registerRoute(route);
+  }
+};