@memlab/lens 1.0.0

package/README.md

@@ -0,0 +1,63 @@
+# MemLens
+
+## What is this?
+
+MemLens is a debugging tool that helps identify memory leaks in React applications. It tracks:
+
+- Detached DOM elements that are no longer connected to the document but still held in memory
+- Unmounted React Fiber nodes that haven't been properly cleaned up
+- Memory usage patterns and growth over time
+
+The tool provides:
+
+1. Real-time memory monitoring through the browser console
+2. Visual representation of problematic DOM elements and React components
+3. Memory usage statistics and trends
+
+This can help developers identify:
+- Components that aren't properly cleaned up
+
+## How to Build?
+
+```bash
+webpack
+```
+
+## How to Test?
+
+1. Install Playwright
+
+```bash
+npx playwright install
+npx playwright install-deps
+```
+
+2. Run the test
+
+```bash
+npm run test:e2e
+```
+
+3. Test manually by copying and pasting the content of `dist/run.bundle.js`
+into web console
+
+
+## TODO List
+ * Note that document.querySelectorAll('*') only captures DOM elements on the DOM tree
+   * So the DOM tree scanning and component name analysis must be done in a frequent interval (every 10s or so)
+ * Extensible framework for tracking additional metadata
+ * Auto diffing and counting detached Element and unmounted Fiber nodes
+   * being able to summarize the leaked components that was not reported (this can reduce the report overhead)
+ * Improve the DOM visualizer - only visualize the common ancestors of the detached DOM elements and unmounted fiber nodes
+ * Improving the scanning efficiency so that the overhead is minimal in production environment
+ * Improve the fiber tree traversal efficiency (there are redundant traversals right now)
+ * Not only keep track of detached DOM elements, but also keep track of unmounted fiber nodes in WeakMap and WeakSet
+ * DOM visualizer is leaking canvas DOM elements after each scan
+ * Monitor event listener leaks?
+ * Real-time memory usage graphs
+ * Real-time component count and other react memory scan obtained stats graphs
+ * Component re-render heat maps
+ * Interactive component tree navigation
+ * Browser extension integration
+ * centralized config file
+ * centralized exception handling

package/dist/index.js

@@ -0,0 +1,112 @@
+/******/ (() => { // webpackBootstrap
+/******/ 	"use strict";
+/******/ 	var __webpack_modules__ = ({
+
+/***/ 3:
+/***/ ((module) => {
+
+module.exports = require("path");
+
+/***/ }),
+
+/***/ 383:
+/***/ ((module) => {
+
+module.exports = require("fs");
+
+/***/ }),
+
+/***/ 783:
+/***/ (function(__unused_webpack_module, exports, __webpack_require__) {
+
+
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", ({ value: true }));
+exports.getBundleContent = getBundleContent;
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall memory_lab
+ */
+const fs = __importStar(__webpack_require__(383));
+const path = __importStar(__webpack_require__(3));
+function getBundleContent() {
+    const bundlePath = path.join(__dirname, 'memlens.run.bundle.min.js');
+    return fs.readFileSync(bundlePath, 'utf-8');
+}
+
+
+/***/ })
+
+/******/ 	});
+/************************************************************************/
+/******/ 	// The module cache
+/******/ 	var __webpack_module_cache__ = {};
+/******/ 	
+/******/ 	// The require function
+/******/ 	function __webpack_require__(moduleId) {
+/******/ 		// Check if module is in cache
+/******/ 		var cachedModule = __webpack_module_cache__[moduleId];
+/******/ 		if (cachedModule !== undefined) {
+/******/ 			return cachedModule.exports;
+/******/ 		}
+/******/ 		// Create a new module (and put it into the cache)
+/******/ 		var module = __webpack_module_cache__[moduleId] = {
+/******/ 			// no module.id needed
+/******/ 			// no module.loaded needed
+/******/ 			exports: {}
+/******/ 		};
+/******/ 	
+/******/ 		// Execute the module function
+/******/ 		__webpack_modules__[moduleId].call(module.exports, module, module.exports, __webpack_require__);
+/******/ 	
+/******/ 		// Return the exports of the module
+/******/ 		return module.exports;
+/******/ 	}
+/******/ 	
+/************************************************************************/
+/******/ 	
+/******/ 	// startup
+/******/ 	// Load entry module and return exports
+/******/ 	// This entry module is referenced by other modules so it can't be inlined
+/******/ 	var __webpack_exports__ = __webpack_require__(783);
+/******/ 	module.exports = __webpack_exports__;
+/******/ 	
+/******/ })()
+;

package/explainer.md

@@ -0,0 +1,54 @@
+
+
+### Purpose and Core Functionality
+This project is a specialized production memory diagnostic collection and local dev debugging tool designed to detect and visualize memory leaks in React applications, with a particular focus on identifying detached components that should have been garbage collected but remain in memory.
+
+### How It Works
+
+#### Detection System
+The module operates through multiple layers of monitoring:
+ * **DOM Scanning**: Continuously scans the Document Object Model to create a comprehensive map of all elements present in the application.
+ * **React Fiber Analysis**: Specifically tracks React's internal fiber tree structure, which represents the component hierarchy and their relationships.
+ * **Memory Leak Detection**: Identifies components that are no longer attached to the main DOM tree but still persist in memory, indicating potential memory leaks.
+
+#### Monitoring Process
+The system employs a dual-monitoring approach:
+ * **Periodic Scanning**: Runs regular scans at configurable intervals to check for detached components.
+ * **Real-time Observation**: Uses mutation observers to detect DOM changes as they happen, ensuring immediate detection of potential issues.
+
+#### Visualization System
+The visualization component provides real-time feedback through:
+ * **Overlay Layer**: Creates a transparent canvas overlay that sits above the application.
+ * **Visual Indicators**: Draws highlighting boxes around problematic components.
+ * **Component Information**: Displays relevant information about detected issues, including component names and their locations.
+
+### Key Features
+
+#### Memory Safety
+- Implements memory-safe tracking mechanisms using weak references
+- Avoids creating memory leaks while detecting them
+- Ensures the debugging tool itself doesn't impact application performance
+
+#### Non-Intrusive Design
+- Operates without interfering with the application's normal functionality
+- Uses a transparent overlay that doesn't block user interactions
+- Can be easily enabled or disabled as needed
+
+#### Developer Tools Integration
+- Provides detailed debugging information in development mode
+- Offers subscription capabilities for external monitoring tools
+- Maintains detailed statistics about detected issues
+
+### Use Cases
+
+ * **Production Performance Monitoring**
+   - Monitoring application memory usage patterns
+   - Detecting gradual memory leaks in long-running applications
+   - Identifying patterns in component detachment
+
+ * **Development Debugging**
+   - Identifying component cleanup issues during development
+   - Tracking down memory leaks in complex component hierarchies
+   - Visualizing component lifecycle issues
+
+This tool serves as an essential debugging aid for React developers, helping them maintain efficient and leak-free applications while providing valuable insights into component behavior and memory management patterns.

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@memlab/lens",
+  "version": "1.0.0",
+  "license": "MIT",
+  "description": "MemLens is a tool for inspecting memory leaks in browser",
+  "author": "Liang Gong <lgong@meta.com>",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "webpack",
+    "rebuild": "webpack",
+    "reset-build": "rm package-lock.json; rm -rf node_modules; npm i; webpack",
+    "test": "playwright test --output=out",
+    "test:manual": "node src/tests/manual/playwright-open-manual.js todo-list/todo-with-run.bundle.html",
+    "test:e2e": "playwright test --output=out",
+    "test:rebuild": "webpack && playwright test --output=out",
+    "build-pkg": "webpack",
+    "publish-patch": "npm publish",
+    "clean-pkg": "rm -rf ./dist && rm -rf ./node_modules && rm -f ./tsconfig.tsbuildinfo && rm -rf out"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.49.1",
+    "@types/react": "^18.3.12",
+    "@types/react-reconciler": "^0.28.8",
+    "react-reconciler": "^0.31.0",
+    "ts-loader": "^9.5.1",
+    "typescript": "^5.7.2",
+    "webpack": "^5.97.1",
+    "webpack-cli": "^5.1.4"
+  }
+}

package/playwright.config.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall memory_lab
+ */
+import process from 'process';
+import {PlaywrightTestConfig} from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  testDir: './src/tests',
+  use: {
+    // Serve files from the root directory
+    baseURL: `file://${process.cwd()}`,
+  },
+};
+
+export default config;

package/src/config/config.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall memory_lab
+ */
+
+import {AnyValue, Config} from '../core/types';
+
+// Performance Configuration
+export const performanceConfig = {
+  scanIntervalMs: 1000,
+  maxComponentStackDepth: 100,
+  memoryMeasurementIntervalMs: 5000,
+};
+
+// Feature Flags
+export const featureFlags = {
+  enableMutationObserver: true,
+  enableMemoryTracking: true,
+  enableComponentStack: true,
+  enableConsoleLogs: (window as AnyValue)?.TEST_MEMORY_SCAN,
+};
+
+// overall Config
+export const config: Config = {
+  performance: performanceConfig,
+  features: featureFlags,
+};

package/src/core/dom-observer.ts

@@ -0,0 +1,189 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall memory_lab
+ */
+import {isWeakAPINative} from '../utils/weak-ref-utils';
+import {isVisualizerElement} from '../visual/visual-utils';
+import {DOMElementStats, DOMObserveCallback, Optional} from './types';
+
+// if false, we only capture the top-most element of a removed subtree
+const CAPTURE_ALL_REMOVED_ELEMENTS = true;
+const IS_WEAK_REF_NATIVE = isWeakAPINative();
+const IS_MUTATION_OBSERVER_SUPPORTED = window.MutationObserver !== undefined;
+
+export class DOMObserver {
+  #elementCount: number;
+  #detachedElementCount: number;
+  #mutationObserver: Optional<MutationObserver>;
+  #trackedElements: Array<WeakRef<Element>>;
+  #trackedElementSet: WeakSet<Element>;
+  #consolidateInterval: Optional<number>;
+  #observeCallback: Array<DOMObserveCallback>;
+
+  constructor() {
+    this.#elementCount = 0;
+    this.#detachedElementCount = 0;
+    this.#trackedElements = [];
+    this.#trackedElementSet = new WeakSet();
+    this.#observeCallback = [];
+    this.startMonitoring();
+  }
+
+  register(callback: DOMObserveCallback): void {
+    this.#observeCallback.push(callback);
+  }
+
+  startMonitoring(): void {
+    if (!IS_WEAK_REF_NATIVE || !IS_MUTATION_OBSERVER_SUPPORTED) {
+      return;
+    }
+    if (this.#mutationObserver != null) {
+      return;
+    }
+    this.#mutationObserver = new MutationObserver(mutationsList => {
+      let newlyAdded: Array<WeakRef<Element>> = [];
+      const updateCallback = (node: Node) => {
+        if (node == null) {
+          return;
+        }
+        if (node.nodeType != Node.ELEMENT_NODE) {
+          return;
+        }
+        const element = node as Element;
+        if (isVisualizerElement(element)) {
+          return;
+        }
+        if (CAPTURE_ALL_REMOVED_ELEMENTS) {
+          const diff = this.#gatherAllElementsInRemovedSubtree(element);
+          newlyAdded = [...newlyAdded, ...diff];
+        } else if (!this.#trackedElementSet.has(element)) {
+          const ref = new WeakRef(element);
+          this.#trackedElements.push(ref);
+          this.#trackedElementSet.add(element);
+          newlyAdded.push(ref);
+        }
+      };
+
+      mutationsList.forEach(mutation => {
+        mutation.addedNodes.forEach(updateCallback);
+        mutation.removedNodes.forEach(updateCallback);
+      });
+
+      this.#observeCallback.forEach(cb => cb(newlyAdded));
+    });
+
+    const waitForBodyAndObserve = () => {
+      if (document.body) {
+        // observe changes in DOM tree
+        this.#mutationObserver?.observe(document.body, {
+          childList: true, // Detect direct additions/removals
+          subtree: true, // Observe all descendants
+        });
+      } else {
+        setTimeout(waitForBodyAndObserve, 0);
+      }
+    };
+
+    waitForBodyAndObserve();
+
+    // starts consolidating removedElements weak references;
+    this.#consolidateInterval = window.setInterval(() => {
+      this.#consolidateElementRefs();
+    }, 3000);
+  }
+
+  stopMonitoring(): void {
+    if (this.#mutationObserver != null) {
+      this.#mutationObserver.disconnect();
+      this.#mutationObserver = null;
+    }
+    if (this.#consolidateInterval != null) {
+      window.clearInterval(this.#consolidateInterval);
+      this.#consolidateInterval = null;
+    }
+    // TODO: clean up memory
+  }
+
+  #consolidateElementRefs(): void {
+    const consolidatedList = [];
+    const trackedElements = new Set<Element>();
+    for (const ref of this.#trackedElements) {
+      const element = ref.deref();
+      if (element != null && !trackedElements.has(element)) {
+        consolidatedList.push(ref);
+        trackedElements.add(element);
+      }
+    }
+    this.#trackedElements = consolidatedList;
+  }
+
+  #getTotalDOMElementCount(): number {
+    return (this.#elementCount =
+      document?.getElementsByTagName('*')?.length ?? 0);
+  }
+
+  #getDetachedElementCount(): number {
+    let count = 0;
+    for (const ref of this.#trackedElements) {
+      const element = ref.deref();
+      if (element && element.isConnected === false) {
+        ++count;
+      }
+    }
+    return (this.#detachedElementCount = count);
+  }
+
+  #gatherAllElementsInRemovedSubtree(node: Node): Array<WeakRef<Element>> {
+    const queue = [node];
+    const visited = new Set<Node>();
+    const newlyAdded: Array<WeakRef<Element>> = [];
+    while (queue.length > 0) {
+      const current = queue.pop();
+      if (current == null || visited.has(current)) {
+        continue;
+      }
+      if (current?.nodeType !== Node.ELEMENT_NODE) {
+        continue;
+      }
+      const element = current as Element;
+      if (isVisualizerElement(element)) {
+        continue;
+      }
+      visited.add(element);
+      if (!this.#trackedElementSet.has(element)) {
+        const ref = new WeakRef(element);
+        this.#trackedElements.push(ref);
+        this.#trackedElementSet.add(element);
+        newlyAdded.push(ref);
+      }
+      const list = element.childNodes;
+      for (let i = 0; i < list.length; ++i) {
+        queue.push(list[i]);
+      }
+    }
+    return newlyAdded;
+  }
+
+  getDOMElements(): Array<WeakRef<Element>> {
+    return [...this.#trackedElements];
+  }
+
+  getStats(): DOMElementStats {
+    try {
+      this.#elementCount = this.#getTotalDOMElementCount();
+      this.#detachedElementCount = this.#getDetachedElementCount();
+    } catch (ex) {
+      // do nothing
+    }
+
+    return {
+      elements: this.#elementCount,
+      detachedElements: this.#detachedElementCount,
+    };
+  }
+}

package/src/core/event-listener-tracker.ts

@@ -0,0 +1,171 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall memory_lab
+ */
+import type {Fiber} from 'react-reconciler';
+import {getFiberNodeFromElement} from '../utils/react-fiber-utils';
+import {WeakMapPlus} from '../utils/weak-ref-utils';
+
+type EventListenerEntry = {
+  type: string;
+  cb: WeakRef<EventListenerOrEventListenerObject>;
+  options?: boolean | AddEventListenerOptions;
+  fiber?: WeakRef<Fiber>;
+};
+
+type DetachedListenerGroup = {
+  type: string;
+  count: number;
+  entries: WeakRef<EventListenerEntry>[];
+};
+
+export class EventListenerTracker {
+  private static instance: EventListenerTracker | null = null;
+  #listenerMap: WeakMapPlus<EventTarget, EventListenerEntry[]>;
+  #detachedListeners: Map<string, DetachedListenerGroup[]>;
+  #originalAddEventListener: typeof EventTarget.prototype.addEventListener;
+  #originalRemoveEventListener: typeof EventTarget.prototype.removeEventListener;
+
+  private constructor() {
+    this.#listenerMap = new WeakMapPlus({fallback: 'noop', cleanupMs: 100});
+    this.#detachedListeners = new Map();
+    this.#originalAddEventListener = EventTarget.prototype.addEventListener;
+    this.#originalRemoveEventListener =
+      EventTarget.prototype.removeEventListener;
+    this.#patchEventListeners();
+  }
+
+  static getInstance(): EventListenerTracker {
+    if (!EventListenerTracker.instance) {
+      EventListenerTracker.instance = new EventListenerTracker();
+    }
+    return EventListenerTracker.instance;
+  }
+
+  #patchEventListeners(): void {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const self = this;
+    EventTarget.prototype.addEventListener = function (
+      type: string,
+      listener: EventListenerOrEventListenerObject,
+      options?: boolean | AddEventListenerOptions,
+    ) {
+      self.#originalAddEventListener.call(this, type, listener, options);
+      if (this instanceof Element) {
+        const fiber = getFiberNodeFromElement(this);
+        const entry: EventListenerEntry = {
+          type,
+          cb: new WeakRef(listener),
+          options,
+          fiber: fiber ? new WeakRef(fiber) : undefined,
+        };
+        const listeners = self.#listenerMap.get(this) ?? [];
+        listeners.push(entry);
+        self.#listenerMap.set(this, listeners);
+      }
+    };
+
+    EventTarget.prototype.removeEventListener = function (
+      type: string,
+      listener: EventListenerOrEventListenerObject,
+      options?: boolean | EventListenerOptions,
+    ) {
+      self.#originalRemoveEventListener.call(this, type, listener, options);
+      if (this instanceof Element) {
+        const listeners = self.#listenerMap.get(this);
+        if (listeners) {
+          const index = listeners.findIndex(
+            entry =>
+              entry.type === type &&
+              entry.cb.deref() === listener &&
+              entry.options === options,
+          );
+          if (index !== -1) {
+            listeners.splice(index, 1);
+          }
+          if (listeners.length === 0) {
+            self.#listenerMap.delete(this);
+          } else {
+            self.#listenerMap.set(this, listeners);
+          }
+        }
+      }
+    };
+  }
+
+  #unpatchEventListeners(): void {
+    EventTarget.prototype.addEventListener = this.#originalAddEventListener;
+    EventTarget.prototype.removeEventListener =
+      this.#originalRemoveEventListener;
+  }
+
+  addListener(
+    el: EventTarget,
+    type: string,
+    cb: EventListenerOrEventListenerObject,
+    options?: boolean | AddEventListenerOptions,
+  ): void {
+    el.addEventListener(type, cb, options);
+  }
+
+  removeListener(
+    el: EventTarget,
+    type: string,
+    cb: EventListenerOrEventListenerObject,
+    options?: boolean | AddEventListenerOptions,
+  ): void {
+    el.removeEventListener(type, cb, options);
+  }
+
+  scan(
+    getComponentName: (elRef: WeakRef<Element>) => string,
+  ): Map<string, DetachedListenerGroup[]> {
+    const detachedListeners = new Map<string, DetachedListenerGroup[]>();
+
+    for (const [el, listeners] of this.#listenerMap.entries()) {
+      if (el instanceof Element && !el.isConnected) {
+        for (const listener of listeners) {
+          // Skip if the callback has been garbage collected
+          if (!listener.cb.deref()) continue;
+
+          const componentName = getComponentName(new WeakRef(el));
+          if (!detachedListeners.has(componentName)) {
+            detachedListeners.set(componentName, []);
+          }
+
+          const groups = detachedListeners.get(componentName);
+          let group = groups?.find(g => g.type === listener.type);
+          if (!group) {
+            group = {
+              type: listener.type,
+              count: 0,
+              entries: [],
+            };
+            groups?.push(group);
+          }
+          group.count++;
+          group.entries.push(new WeakRef(listener));
+        }
+      }
+    }
+
+    this.#detachedListeners = detachedListeners;
+    return detachedListeners;
+  }
+
+  getDetachedListeners(): Map<string, DetachedListenerGroup[]> {
+    return this.#detachedListeners;
+  }
+
+  destroy(): void {
+    this.#unpatchEventListeners();
+    this.#listenerMap.destroy();
+    this.#detachedListeners.clear();
+    EventListenerTracker.instance = null;
+  }
+}