@memlab/core 1.1.16 → 1.1.17

package/dist/lib/Config.d.ts

@@ -71,6 +71,7 @@ export declare class MemLabConfig {
     userDataDir: string;
     curDataDir: string;
     webSourceDir: string;
+    debugDataDir: string;
     runMetaFile: string;
     snapshotSequenceFile: string;
     exploreResultFile: string;
@@ -90,6 +91,7 @@ export declare class MemLabConfig {
     traceClusterOutDir: string;
     traceJsonOutDir: string;
     metricsOutDir: string;
+    heapAnalysisLogDir: string;
     reportScreenshotFile: string;
     newUniqueClusterDir: string;
     staleUniqueClusterDir: string;

package/dist/lib/Console.d.ts

@@ -23,6 +23,7 @@ declare class MemLabConsole {
     private config;
     private sections;
     private log;
+    private logFileSet;
     private styles;
     private static singleton;
     protected constructor();
@@ -40,6 +41,8 @@ declare class MemLabConsole {
     private shouldBeConcise;
     private clearPrevOverwriteMsg;
     private printStr;
+    registerLogFile(logFile: string): void;
+    unregisterLogFile(logFile: string): void;
     beginSection(name: string): void;
     endSection(name: string): void;
     setConfig(config: MemLabConfig): void;

package/dist/lib/Console.js

@@ -14,10 +14,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const chalk_1 = __importDefault(require("chalk"));
 const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
 const string_width_1 = __importDefault(require("string-width"));
 const stdout = process.stdout;
 const TABLE_MAX_WIDTH = 50;
+const LOG_BUFFER_LENGTH = 100;
 const prevLine = '\x1b[F';
 const eraseLine = '\x1b[K';
 const barComplete = chalk_1.default.green('\u2588');
@@ -42,10 +44,21 @@ function formatTableArg(arg) {
         }
     });
 }
+function registerExitCleanup(inst, exitHandler) {
+    const p = process;
+    // normal exit
+    p.on('exit', exitHandler.bind(null, { cleanup: true }));
+    // ctrl + c event
+    p.on('SIGINT', exitHandler.bind(null, { exit: true }));
+    // kill pid
+    p.on('SIGUSR1', exitHandler.bind(null, { exit: true }));
+    p.on('SIGUSR2', exitHandler.bind(null, { exit: true }));
+}
 class MemLabConsole {
     constructor() {
         this.config = {};
         this.log = [];
+        this.logFileSet = new Set();
         this.styles = {
             top: (msg) => msg,
             high: chalk_1.default.dim.bind(chalk_1.default),
@@ -67,12 +80,15 @@ class MemLabConsole {
         }
         const inst = new MemLabConsole();
         MemLabConsole.singleton = inst;
-        // clean up output
+        const exitHandler = (
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        _options, 
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        process.on('exit', (_code) => {
-            inst.flushLog();
+        _exitCode) => {
+            inst.flushLog({ sync: true });
             inst.clearPrevOverwriteMsg();
-        });
+        };
+        registerExitCleanup(inst, exitHandler);
         return inst;
     }
     style(msg, name) {
@@ -98,28 +114,49 @@ class MemLabConsole {
     }
     logMsg(msg) {
         // remove control characters
-        const rawMsg = msg
+        const lines = msg.split('\n').map(line => line
             // eslint-disable-next-line no-control-regex
             .replace(/[\u0000-\u001F\u007F-\u009F]/g, '')
-            .replace(/\[\d{1,3}m/g, '');
-        this.log.push(rawMsg);
-        if (this.log.length > 20) {
-            this.flushLog();
+            .replace(/\[\d{1,3}m/g, ''));
+        this.log.push(...lines);
+        if (this.log.length > LOG_BUFFER_LENGTH) {
+            this.flushLog({ sync: true });
         }
     }
-    flushLog() {
+    flushLog(options = {}) {
         const str = this.log.join('\n');
-        if (str.length > 0) {
-            const file = this.config.consoleLogFile;
-            fs_1.default.appendFile(file, str + '\n', 'UTF-8', () => {
-                // NOOP
-            });
-        }
         this.log = [];
+        if (str.length === 0) {
+            return;
+        }
+        // synchronous logging
+        if (options.sync) {
+            for (const logFile of this.logFileSet) {
+                try {
+                    fs_1.default.appendFileSync(logFile, str + '\n', 'UTF-8');
+                }
+                catch (_a) {
+                    // fail silently
+                }
+            }
+        }
+        else {
+            // async logging
+            const emptyCallback = () => {
+                // no op
+            };
+            for (const logFile of this.logFileSet) {
+                try {
+                    fs_1.default.appendFile(logFile, str + '\n', 'UTF-8', emptyCallback);
+                }
+                catch (_b) {
+                    // fail silently
+                }
+            }
+        }
     }
     pushMsg(msg, options = {}) {
-        const len = this.sections.arr.length;
-        if (this.config.isContinuousTest || len === 0) {
+        if (this.sections.arr.length === 0) {
             return;
         }
         // calculate each line's visible width
@@ -140,7 +177,9 @@ class MemLabConsole {
         if (!section || section.msgs.length === 0) {
             return;
         }
-        stdout.write(eraseLine);
+        if (!this.config.muteConsole) {
+            stdout.write(eraseLine);
+        }
         const msg = section.msgs.pop();
         if (!msg) {
             return;
@@ -150,8 +189,10 @@ class MemLabConsole {
             const line = (_a = lines.pop()) !== null && _a !== void 0 ? _a : 0;
             const width = stdout.columns;
             let n = line === 0 ? 1 : Math.ceil(line / width);
-            while (n-- > 0) {
-                stdout.write(prevLine + eraseLine);
+            if (!this.config.muteConsole && !this.config.isTest) {
+                while (n-- > 0) {
+                    stdout.write(prevLine + eraseLine);
+                }
             }
         }
     }
@@ -189,11 +230,21 @@ class MemLabConsole {
         this.clearPrevMsgInLastSection();
     }
     printStr(msg, options = {}) {
-        if (this.config.isTest || this.config.muteConsole) {
+        this.pushMsg(msg, options);
+        if (this.config.isTest) {
             return;
         }
-        console.log(msg);
-        this.pushMsg(msg, options);
+        if (this.config.isContinuousTest || !this.config.muteConsole) {
+            console.log(msg);
+        }
+    }
+    registerLogFile(logFile) {
+        this.flushLog({ sync: true });
+        this.logFileSet.add(path_1.default.resolve(logFile));
+    }
+    unregisterLogFile(logFile) {
+        this.flushLog({ sync: true });
+        this.logFileSet.delete(path_1.default.resolve(logFile));
     }
     beginSection(name) {
         if (this.config.isContinuousTest) {
@@ -300,14 +351,16 @@ class MemLabConsole {
         this.printStr('');
     }
     overwrite(msg, options = {}) {
-        if (this.config.isTest || this.config.muteConsole) {
+        const str = this.style(msg, options.level || 'low');
+        if (this.config.isContinuousTest) {
+            this.printStr(msg, { isOverwrite: false });
             return;
         }
-        if (this.config.isContinuousTest) {
-            return console.log(msg);
+        if (this.config.isTest || this.config.muteConsole) {
+            this.printStr(str, { isOverwrite: true });
+            return;
         }
         this.clearPrevOverwriteMsg();
-        const str = this.style(msg, options.level || 'low');
         this.printStr(str, { isOverwrite: true });
     }
     waitForConsole(query) {

package/dist/lib/FileManager.d.ts

@@ -32,8 +32,11 @@ export declare class FileManager {
     getCurDataDir(options: FileOption): string;
     getWebSourceDir(options?: FileOption): string;
     getWebSourceMetaFile(options?: FileOption): string;
+    getDebugDataDir(options?: FileOption): string;
+    getDebugSourceFile(options?: FileOption): string;
     getPersistDataDir(options: FileOption): string;
     getLoggerOutDir(options?: FileOption): string;
+    getHeapAnalysisLogDir(options?: FileOption): string;
     getTraceClustersDir(options?: FileOption): string;
     getTraceJSONDir(options?: FileOption): string;
     getUnclassifiedTraceClusterDir(options?: FileOption): string;
@@ -61,6 +64,7 @@ export declare class FileManager {
         controlWorkDir: string;
         testWorkDir: string;
     };
+    initNewHeapAnalysisLogFile(options?: FileOption): string;
     getAndInitTSCompileIntermediateDir(): string;
     clearDataDirs(options?: FileOption): void;
     emptyDirIfExists(dir: string): void;

package/dist/lib/FileManager.js

@@ -98,12 +98,22 @@ class FileManager {
     getWebSourceMetaFile(options = {}) {
         return path_1.default.join(this.getWebSourceDir(options), 'files.json');
     }
+    getDebugDataDir(options = {}) {
+        return path_1.default.join(this.getDataBaseDir(options), 'debug');
+    }
+    getDebugSourceFile(options = {}) {
+        return path_1.default.join(this.getDebugDataDir(options), 'file.js');
+    }
     getPersistDataDir(options) {
         return path_1.default.join(this.getDataBaseDir(options), 'persist');
     }
     getLoggerOutDir(options = {}) {
         return path_1.default.join(this.getDataBaseDir(options), 'logger');
     }
+    // all heap analysis results generated
+    getHeapAnalysisLogDir(options = {}) {
+        return path_1.default.join(this.getLoggerOutDir(options), 'heap-analysis');
+    }
     // all trace clusters generated from the current run
     getTraceClustersDir(options = {}) {
         return path_1.default.join(this.getLoggerOutDir(options), 'trace-clusters');
@@ -200,6 +210,15 @@ class FileManager {
         const testWorkDir = joinAndProcessDir({}, expDir, 'test');
         return { controlWorkDir, testWorkDir };
     }
+    // create a unique log file created for heap analysis output
+    initNewHeapAnalysisLogFile(options = {}) {
+        const dir = this.getHeapAnalysisLogDir(options);
+        const file = path_1.default.join(dir, `analysis-${Utils_1.default.getUniqueID()}-out.log`);
+        if (!fs_extra_1.default.existsSync(file)) {
+            fs_extra_1.default.createFileSync(file);
+        }
+        return file;
+    }
     getAndInitTSCompileIntermediateDir() {
         const dir = path_1.default.join(this.getTmpDir(), 'memlab-code');
         this.rmDir(dir);
@@ -212,6 +231,7 @@ class FileManager {
             return;
         }
         this.emptyDirIfExists(this.getWebSourceDir(options));
+        this.emptyDirIfExists(this.getDebugDataDir(options));
         const dataSuffix = ['.heapsnapshot', '.json', '.png'];
         const files = fs_extra_1.default.readdirSync(curDataDir);
         for (const file of files) {
@@ -238,6 +258,8 @@ class FileManager {
         this.emptyDirIfExists(this.getUnclassifiedTraceClusterDir(options));
         // all unique cluster info
         this.emptyDirIfExists(this.getUniqueTraceClusterDir(options));
+        // all heap analysis results
+        this.emptyDirIfExists(this.getHeapAnalysisLogDir(options));
     }
     resetBrowserDir() {
         try {
@@ -294,9 +316,12 @@ class FileManager {
         const outDir = joinAndProcessDir(options, this.getDataOutDir(options));
         config.curDataDir = joinAndProcessDir(options, this.getCurDataDir(options));
         config.webSourceDir = joinAndProcessDir(options, this.getWebSourceDir(options));
+        config.debugDataDir = joinAndProcessDir(options, this.getDebugDataDir(options));
         config.dataBuilderDataDir = joinAndProcessDir(options, config.dataBaseDir, 'dataBuilder');
         config.persistentDataDir = joinAndProcessDir(options, this.getPersistDataDir(options));
+        // register the default log file
         config.consoleLogFile = path_1.default.join(config.curDataDir, 'console-log.txt');
+        Console_1.default.registerLogFile(config.consoleLogFile);
         config.runMetaFile = this.getRunMetaFile(options);
         config.snapshotSequenceFile = this.getSnapshotSequenceMetaFile(options);
         config.browserInfoSummary = path_1.default.join(config.curDataDir, 'browser-info.txt');
@@ -314,6 +339,8 @@ class FileManager {
         config.traceClusterOutDir = joinAndProcessDir(options, this.getTraceClustersDir(options));
         // detailed trace json files for visualization
         config.traceJsonOutDir = joinAndProcessDir(options, this.getTraceJSONDir(options));
+        // heap analysis results
+        config.heapAnalysisLogDir = joinAndProcessDir(options, this.getHeapAnalysisLogDir(options));
         config.metricsOutDir = joinAndProcessDir(options, loggerOutDir, 'metrics');
         config.reportScreenshotFile = path_1.default.join(outDir, 'report.png');
         const codeDataDir = this.getCodeDataDir();

package/dist/lib/HeapAnalyzer.d.ts

@@ -7,15 +7,10 @@
  * @format
  * @oncall web_perf_infra
  */
-import type { E2EStepInfo, HeapNodeIdSet, IHeapNode, IHeapSnapshot, IMemoryAnalystOptions, IMemoryAnalystSnapshotDiff, LeakTracePathItem, Optional, IOveralLeakInfo, TraceCluster, ISerializedInfo } from './Types';
+import type { E2EStepInfo, HeapNodeIdSet, IHeapSnapshot, IMemoryAnalystOptions, IMemoryAnalystSnapshotDiff, IOveralHeapInfo, LeakTracePathItem, Optional, IOveralLeakInfo, TraceCluster, ISerializedInfo } from './Types';
 import TraceFinder from '../paths/TraceFinder';
 declare class MemoryAnalyst {
     checkLeak(): Promise<ISerializedInfo[]>;
-    checkUnbound(options?: IMemoryAnalystOptions): Promise<void>;
-    breakDownMemoryByShapes(options?: {
-        file?: string;
-    }): Promise<void>;
-    detectUnboundGrowth(options?: IMemoryAnalystOptions): Promise<void>;
     detectMemoryLeaks(): Promise<ISerializedInfo[]>;
     visualizeMemoryUsage(options?: IMemoryAnalystOptions): void;
     focus(options?: {
@@ -23,19 +18,15 @@ declare class MemoryAnalyst {
     }): Promise<void>;
     shouldLoadCompleteSnapshot(tabsOrder: E2EStepInfo[], tab: E2EStepInfo): boolean;
     diffSnapshots(loadAll?: boolean): Promise<IMemoryAnalystSnapshotDiff>;
-    private calculateRetainedSizes;
     preparePathFinder(snapshot: IHeapSnapshot): TraceFinder;
     private dumpPageInteractionSummary;
     private dumpLeakSummaryToConsole;
     private filterLeakedObjects;
-    aggregateDominatorMetrics(ids: HeapNodeIdSet, snapshot: IHeapSnapshot, checkNodeCb: (node: IHeapNode) => boolean, nodeMetricsCb: (node: IHeapNode) => number): number;
-    private getOverallHeapInfo;
-    getRetainedSize(node: IHeapNode): number;
+    getOverallHeapInfo(snapshot: IHeapSnapshot, options?: {
+        force?: boolean;
+    }): Optional<IOveralHeapInfo>;
     getOverallLeakInfo(leakedNodeIds: HeapNodeIdSet, snapshot: IHeapSnapshot): Optional<IOveralLeakInfo>;
-    private printHeapInfo;
-    private breakDownSnapshotByShapes;
-    private isTrivialEdgeForBreakDown;
-    private breakDownByReferrers;
+    printHeapInfo(leakInfo: IOveralHeapInfo): void;
     private printHeapAndLeakInfo;
     private logLeakTraceSummary;
     searchLeakedTraces(leakedNodeIds: HeapNodeIdSet, snapshot: IHeapSnapshot): Promise<{

package/dist/lib/HeapAnalyzer.js

@@ -23,7 +23,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const babar_1 = __importDefault(require("babar"));
-const chalk_1 = __importDefault(require("chalk"));
 const LeakClusterLogger_1 = __importDefault(require("../logger/LeakClusterLogger"));
 const LeakTraceDetailsLogger_1 = __importDefault(require("../logger/LeakTraceDetailsLogger"));
 const TraceFinder_1 = __importDefault(require("../paths/TraceFinder"));
@@ -42,156 +41,6 @@ class MemoryAnalyst {
             return yield this.detectMemoryLeaks();
         });
     }
-    checkUnbound(options = {}) {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.visualizeMemoryUsage(options);
-            Utils_1.default.checkSnapshots(options);
-            yield this.detectUnboundGrowth(options);
-        });
-    }
-    breakDownMemoryByShapes(options = {}) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const opt = { buildNodeIdIndex: true, verbose: true };
-            const file = options.file ||
-                Utils_1.default.getSnapshotFilePathWithTabType(/.*/) ||
-                '<EMPTY_FILE_PATH>';
-            const snapshot = yield Utils_1.default.getSnapshotFromFile(file, opt);
-            this.preparePathFinder(snapshot);
-            const heapInfo = this.getOverallHeapInfo(snapshot, { force: true });
-            if (heapInfo) {
-                this.printHeapInfo(heapInfo);
-            }
-            this.breakDownSnapshotByShapes(snapshot);
-        });
-    }
-    // find any objects that keeps growing
-    detectUnboundGrowth(options = {}) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const nodeInfo = Object.create(null);
-            let hasCheckedFirstSnapshot = false;
-            let snapshot = null;
-            const isValidNode = (node) => node.type === 'object' ||
-                node.type === 'closure' ||
-                node.type === 'regexp';
-            const initNodeInfo = (node) => {
-                if (!isValidNode(node)) {
-                    return;
-                }
-                const n = node.retainedSize;
-                nodeInfo[node.id] = {
-                    type: node.type,
-                    name: node.name,
-                    min: n,
-                    max: n,
-                    history: [n],
-                    node,
-                };
-            };
-            const updateNodeInfo = (node) => {
-                const item = nodeInfo[node.id];
-                if (!item) {
-                    return;
-                }
-                if (node.name !== item.name || node.type !== item.type) {
-                    nodeInfo[node.id] = null;
-                    return;
-                }
-                const n = node.retainedSize;
-                // only monotonic increase?
-                if (Config_1.default.monotonicUnboundGrowthOnly && n < item.max) {
-                    nodeInfo[node.id] = null;
-                    return;
-                }
-                item.history.push(n);
-                item.max = Math.max(item.max, n);
-                item.min = Math.min(item.min, n);
-            };
-            // summarize the heap objects info in current heap snapshot
-            // this is mainly used for better understanding of the % of
-            // objects released and allocated over time
-            const maybeSummarizeNodeInfo = () => {
-                if (!Config_1.default.verbose) {
-                    return;
-                }
-                let n = 0;
-                for (const k in nodeInfo) {
-                    if (nodeInfo[k]) {
-                        ++n;
-                    }
-                }
-                Console_1.default.lowLevel(`Objects tracked: ${n}`);
-            };
-            Console_1.default.overwrite('Checking unbounded objects...');
-            const snapshotFiles = options.snapshotDir
-                ? // load snapshots from a directory
-                    Utils_1.default.getSnapshotFilesInDir(options.snapshotDir)
-                : // load snapshots based on the visit sequence meta data
-                    Utils_1.default.getSnapshotFilesFromTabsOrder();
-            for (const file of snapshotFiles) {
-                // force GC before loading each snapshot
-                if (global.gc) {
-                    global.gc();
-                }
-                // load and preprocess heap snapshot
-                const opt = { buildNodeIdIndex: true, verbose: true };
-                snapshot = yield Utils_1.default.getSnapshotFromFile(file, opt);
-                this.calculateRetainedSizes(snapshot);
-                // keep track of heap objects
-                if (!hasCheckedFirstSnapshot) {
-                    // record Ids in the snapshot
-                    snapshot.nodes.forEach(initNodeInfo);
-                    hasCheckedFirstSnapshot = true;
-                }
-                else {
-                    snapshot.nodes.forEach(updateNodeInfo);
-                    maybeSummarizeNodeInfo();
-                }
-            }
-            // exit if no heap snapshot found
-            if (!hasCheckedFirstSnapshot) {
-                return;
-            }
-            // post process and print the unbounded objects
-            const idsInLastSnapshot = new Set();
-            snapshot === null || snapshot === void 0 ? void 0 : snapshot.nodes.forEach(node => {
-                idsInLastSnapshot.add(node.id);
-            });
-            let ids = [];
-            for (const key in nodeInfo) {
-                const id = parseInt(key, 10);
-                const item = nodeInfo[id];
-                if (!item) {
-                    continue;
-                }
-                if (!idsInLastSnapshot.has(id)) {
-                    continue;
-                }
-                if (item.min === item.max) {
-                    continue;
-                }
-                // filter out non-significant leaks
-                if (item.history[item.history.length - 1] < Config_1.default.unboundSizeThreshold) {
-                    continue;
-                }
-                ids.push(Object.assign({ id }, item));
-            }
-            if (ids.length === 0) {
-                Console_1.default.midLevel('No increasing objects found.');
-                return;
-            }
-            ids = ids
-                .sort((o1, o2) => o2.history[o2.history.length - 1] - o1.history[o1.history.length - 1])
-                .slice(0, 20);
-            // print on terminal
-            const str = Serializer_1.default.summarizeUnboundedObjects(ids, { color: true });
-            Console_1.default.topLevel('Top growing objects in sizes:');
-            Console_1.default.lowLevel(' (Use `memlab trace --node-id=@ID` to get trace)');
-            Console_1.default.topLevel('\n' + str);
-            // save results to file
-            const csv = Serializer_1.default.summarizeUnboundedObjectsToCSV(ids);
-            fs_1.default.writeFileSync(Config_1.default.unboundObjectCSV, csv, 'UTF-8');
-        });
-    }
     // find all unique pattern of leaks
     detectMemoryLeaks() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -368,11 +217,6 @@ class MemoryAnalyst {
             };
         });
     }
-    calculateRetainedSizes(snapshot) {
-        const finder = new TraceFinder_1.default();
-        // dominator and retained size
-        finder.calculateAllNodesRetainedSizes(snapshot);
-    }
     // initialize the path finder
     preparePathFinder(snapshot) {
         const finder = new TraceFinder_1.default();
@@ -442,14 +286,6 @@ class MemoryAnalyst {
             Console_1.default.midLevel(`${leakedNodeIds.size} Fiber nodes and Detached elements`);
         }
     }
-    aggregateDominatorMetrics(ids, snapshot, checkNodeCb, nodeMetricsCb) {
-        let ret = 0;
-        const dominators = Utils_1.default.getConditionalDominatorIds(ids, snapshot, checkNodeCb);
-        Utils_1.default.applyToNodes(dominators, snapshot, node => {
-            ret += nodeMetricsCb(node);
-        });
-        return ret;
-    }
     getOverallHeapInfo(snapshot, options = {}) {
         if (!Config_1.default.verbose && !options.force) {
             return;
@@ -457,22 +293,19 @@ class MemoryAnalyst {
         Console_1.default.overwrite('summarizing heap info...');
         const allIds = Utils_1.default.getNodesIdSet(snapshot);
         const heapInfo = {
-            fiberNodeSize: this.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isFiberNode, this.getRetainedSize),
-            regularFiberNodeSize: this.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isRegularFiberNode, this.getRetainedSize),
-            detachedFiberNodeSize: this.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isDetachedFiberNode, this.getRetainedSize),
-            alternateFiberNodeSize: this.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isAlternateNode, this.getRetainedSize),
-            error: this.aggregateDominatorMetrics(allIds, snapshot, node => node.name === 'Error', this.getRetainedSize),
+            fiberNodeSize: Utils_1.default.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isFiberNode, Utils_1.default.getRetainedSize),
+            regularFiberNodeSize: Utils_1.default.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isRegularFiberNode, Utils_1.default.getRetainedSize),
+            detachedFiberNodeSize: Utils_1.default.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isDetachedFiberNode, Utils_1.default.getRetainedSize),
+            alternateFiberNodeSize: Utils_1.default.aggregateDominatorMetrics(allIds, snapshot, Utils_1.default.isAlternateNode, Utils_1.default.getRetainedSize),
+            error: Utils_1.default.aggregateDominatorMetrics(allIds, snapshot, node => node.name === 'Error', Utils_1.default.getRetainedSize),
         };
         return heapInfo;
     }
-    getRetainedSize(node) {
-        return node.retainedSize;
-    }
     getOverallLeakInfo(leakedNodeIds, snapshot) {
         if (!Config_1.default.verbose) {
             return;
         }
-        const leakInfo = Object.assign(Object.assign({}, this.getOverallHeapInfo(snapshot)), { leakedSize: this.aggregateDominatorMetrics(leakedNodeIds, snapshot, () => true, this.getRetainedSize), leakedFiberNodeSize: this.aggregateDominatorMetrics(leakedNodeIds, snapshot, Utils_1.default.isFiberNode, this.getRetainedSize), leakedAlternateFiberNodeSize: this.aggregateDominatorMetrics(leakedNodeIds, snapshot, Utils_1.default.isAlternateNode, this.getRetainedSize) });
+        const leakInfo = Object.assign(Object.assign({}, this.getOverallHeapInfo(snapshot)), { leakedSize: Utils_1.default.aggregateDominatorMetrics(leakedNodeIds, snapshot, () => true, Utils_1.default.getRetainedSize), leakedFiberNodeSize: Utils_1.default.aggregateDominatorMetrics(leakedNodeIds, snapshot, Utils_1.default.isFiberNode, Utils_1.default.getRetainedSize), leakedAlternateFiberNodeSize: Utils_1.default.aggregateDominatorMetrics(leakedNodeIds, snapshot, Utils_1.default.isAlternateNode, Utils_1.default.getRetainedSize) });
         return leakInfo;
     }
     printHeapInfo(leakInfo) {
@@ -485,111 +318,6 @@ class MemoryAnalyst {
             Console_1.default.topLevel(`· ${name}: ${value}`);
         });
     }
-    breakDownSnapshotByShapes(snapshot) {
-        Console_1.default.overwrite('Breaking down memory by shapes...');
-        const breakdown = Object.create(null);
-        const population = Object.create(null);
-        // group objects based on their shapes
-        snapshot.nodes.forEach(node => {
-            if ((node.type !== 'object' && !Utils_1.default.isStringNode(node)) ||
-                Config_1.default.nodeIgnoreSetInShape.has(node.name)) {
-                return;
-            }
-            const key = Serializer_1.default.summarizeNodeShape(node);
-            breakdown[key] = breakdown[key] || new Set();
-            breakdown[key].add(node.id);
-            if (population[key] === undefined) {
-                population[key] = { examples: [], n: 0 };
-            }
-            ++population[key].n;
-            // retain the top 5 examples
-            const examples = population[key].examples;
-            examples.push(node);
-            examples.sort((n1, n2) => n2.retainedSize - n1.retainedSize);
-            if (examples.length > 5) {
-                examples.pop();
-            }
-        });
-        // calculate and sort based on retained sizes
-        const ret = [];
-        for (const key in breakdown) {
-            const size = this.aggregateDominatorMetrics(breakdown[key], snapshot, () => true, this.getRetainedSize);
-            ret.push({ key, retainedSize: size });
-        }
-        ret.sort((o1, o2) => o2.retainedSize - o1.retainedSize);
-        Console_1.default.topLevel('Object shapes with top retained sizes:');
-        Console_1.default.lowLevel(' (Use `memlab trace --node-id=@ID` to get trace)\n');
-        const topList = ret.slice(0, 40);
-        // print settings
-        const opt = { color: true, compact: true };
-        const dot = chalk_1.default.grey('· ');
-        const colon = chalk_1.default.grey(': ');
-        // print the shapes with the biggest retained size
-        for (const o of topList) {
-            const referrerInfo = this.breakDownByReferrers(breakdown[o.key], snapshot);
-            const { examples, n } = population[o.key];
-            const shapeStr = Serializer_1.default.summarizeNodeShape(examples[0], opt);
-            const bytes = Utils_1.default.getReadableBytes(o.retainedSize);
-            const examplesStr = examples
-                .map(e => `@${e.id} [${Utils_1.default.getReadableBytes(e.retainedSize)}]`)
-                .join(' | ');
-            const meta = chalk_1.default.grey(` (N: ${n}, Examples: ${examplesStr})`);
-            Console_1.default.topLevel(`${dot}${shapeStr}${colon}${bytes}${meta}`);
-            Console_1.default.lowLevel(referrerInfo + '\n');
-        }
-    }
-    isTrivialEdgeForBreakDown(edge) {
-        const source = edge.fromNode;
-        return (source.type === 'array' ||
-            source.name === '(object elements)' ||
-            source.name === 'system' ||
-            edge.name_or_index === '__proto__' ||
-            edge.name_or_index === 'prototype');
-    }
-    breakDownByReferrers(ids, snapshot) {
-        const edgeNames = Object.create(null);
-        for (const id of ids) {
-            const node = snapshot.getNodeById(id);
-            for (const edge of (node === null || node === void 0 ? void 0 : node.referrers) || []) {
-                const source = edge.fromNode;
-                if (!Utils_1.default.isMeaningfulEdge(edge) ||
-                    this.isTrivialEdgeForBreakDown(edge)) {
-                    continue;
-                }
-                const sourceName = Serializer_1.default.summarizeNodeName(source, {
-                    color: false,
-                });
-                const edgeName = Serializer_1.default.summarizeEdgeName(edge, {
-                    color: false,
-                    abstract: true,
-                });
-                const edgeKey = `[${sourceName}] --${edgeName}--> `;
-                edgeNames[edgeKey] = edgeNames[edgeKey] || {
-                    numberOfEdgesToNode: 0,
-                    source,
-                    edge,
-                };
-                ++edgeNames[edgeKey].numberOfEdgesToNode;
-            }
-        }
-        const referrerInfo = Object.entries(edgeNames)
-            .sort((i1, i2) => i2[1].numberOfEdgesToNode - i1[1].numberOfEdgesToNode)
-            .slice(0, 4)
-            .map(i => {
-            const meta = i[1];
-            const source = Serializer_1.default.summarizeNodeName(meta.source, {
-                color: true,
-            });
-            const edgeName = Serializer_1.default.summarizeEdgeName(meta.edge, {
-                color: true,
-                abstract: true,
-            });
-            const edgeSummary = `${source} --${edgeName}-->`;
-            return `  · ${edgeSummary}: ${meta.numberOfEdgesToNode}`;
-        })
-            .join('\n');
-        return referrerInfo;
-    }
     printHeapAndLeakInfo(leakedNodeIds, snapshot) {
         // write page interaction summary to the leaks text file
         this.dumpPageInteractionSummary();
@@ -616,10 +344,6 @@ class MemoryAnalyst {
             this.printHeapAndLeakInfo(leakedNodeIds, snapshot);
             // get all leaked objects
             this.filterLeakedObjects(leakedNodeIds, snapshot);
-            if (Config_1.default.verbose) {
-                // show a breakdown of different object structures
-                this.breakDownSnapshotByShapes(snapshot);
-            }
             const nodeIdInPaths = new Set();
             const paths = [];
             let numOfLeakedObjects = 0;
@@ -642,7 +366,7 @@ class MemoryAnalyst {
                 Console_1.default.midLevel(`${numOfLeakedObjects} leaked objects`);
             }
             // cluster traces from the current run
-            const clusters = TraceBucket_1.default.clusterPaths(paths, snapshot, this.aggregateDominatorMetrics, {
+            const clusters = TraceBucket_1.default.clusterPaths(paths, snapshot, Utils_1.default.aggregateDominatorMetrics, {
                 strategy: Config_1.default.isMLClustering
                     ? new MLTraceSimilarityStrategy_1.default()
                     : undefined,
@@ -651,7 +375,7 @@ class MemoryAnalyst {
             yield this.serializeClusterUpdate(clusters);
             if (Config_1.default.logUnclassifiedClusters) {
                 // cluster traces from the current run
-                const clustersUnclassified = TraceBucket_1.default.generateUnClassifiedClusters(paths, snapshot, this.aggregateDominatorMetrics);
+                const clustersUnclassified = TraceBucket_1.default.generateUnClassifiedClusters(paths, snapshot, Utils_1.default.aggregateDominatorMetrics);
                 LeakClusterLogger_1.default.logUnclassifiedClusters(clustersUnclassified);
             }
             return {
@@ -682,7 +406,7 @@ class MemoryAnalyst {
             }
         }, { reverse: true });
         // cluster traces from the current run
-        const clusters = TraceBucket_1.default.clusterPaths(paths, snapshot, this.aggregateDominatorMetrics, {
+        const clusters = TraceBucket_1.default.clusterPaths(paths, snapshot, Utils_1.default.aggregateDominatorMetrics, {
             strategy: Config_1.default.isMLClustering
                 ? new MLTraceSimilarityStrategy_1.default()
                 : undefined,

package/dist/lib/Types.d.ts

@@ -8,7 +8,7 @@
  * @oncall web_perf_infra
  */
 import { ParsedArgs } from 'minimist';
-import type { LaunchOptions, Page } from 'puppeteer';
+import type { LaunchOptions, Page as PuppeteerPage } from 'puppeteer';
 import type { ErrorHandling, MemLabConfig } from './Config';
 /** @internal */
 export declare type AnyValue = any;
@@ -56,6 +56,12 @@ export declare type XvfbType = {
     display: () => string;
 };
 /** @internal */
+export declare type ShellOptions = {
+    dir?: Optional<string>;
+    ignoreError?: Optional<boolean>;
+    disconnectStdio?: Optional<boolean>;
+};
+/** @internal */
 export declare type CLIArgs = {
     verbose: boolean;
     app: string;
@@ -85,6 +91,57 @@ export declare type CLIArgs = {
     'local-puppeteer': boolean;
     'snapshot-dir': string;
 };
+/**
+ * This is the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
+ * class used by MemLab. The puppeteer `Page` class instance provides
+ * APIs to interact with the web browser.
+ *
+ * The puppeteer `Page` type can be incompatible across different versions.
+ * Your local npm-installed puppeteer version may be different from
+ * the puppeteer used by MemLab. This may cause some type errors, for example:
+ *
+ * ```typescript
+ * import type {Page} from 'puppeteer';
+ * import type {RunOptions} from '@memlab/api';
+ *
+ * const runOptions: RunOptions = {
+ *   scenario: {
+ *     // initial page load url: Google Maps
+ *     url: () => {
+ *       return "https://www.google.com/maps/@37.386427,-122.0428214,11z";
+ *     },
+ *     // type error here if your local puppeeter version is different
+ *     // from the puppeteer used by MemLab
+ *     action: async function (page: Page) {
+ *       await page.click('button[aria-label="Hotels"]');
+ *     },
+ *   },
+ * };
+ * ```
+ *
+ * To avoid the type error in the code example above, MemLab exports the
+ * puppeteer `Page` type used by MemLab so that your code can import it
+ * when necessary:
+ *
+ * ```typescript
+ * import type {Page} from '@memlab/core' // import Page type from memlab
+ * import type {RunOptions} from 'memlab';
+ *
+ * const runOptions: RunOptions = {
+ *   scenario: {
+ *     // initial page load url: Google Maps
+ *     url: () => {
+ *       return "https://www.google.com/maps/@37.386427,-122.0428214,11z";
+ *     },
+ *     // no type error here
+ *     action: async function (page: Page) {
+ *       await page.click('button[aria-label="Hotels"]');
+ *     },
+ *   },
+ * };
+ * ```
+ */
+export declare type Page = PuppeteerPage;
 /**
  * the predicate callback is used to decide if a
  * entity of type `T`.
@@ -338,7 +395,8 @@ export declare type LeakFilterCallback = (node: IHeapNode, snapshot: IHeapSnapsh
  * For concrete examples, check out {@link action} or {@link back}.
  *
  * @param page the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
- * object, which provides APIs to interact with the web browser
+ * object, which provides APIs to interact with the web browser.
+ * To import this type, check out {@link Page}.
  * @returns no return value
  */
 export declare type InteractionsCallback = (page: Page, args?: OperationArgs) => Promise<void>;
@@ -420,7 +478,8 @@ export interface IScenario {
      *
      * * **Parameters**:
      *   * page: `Page` | the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
-     *     object, which provides APIs to interact with the web browser
+     *     object, which provides APIs to interact with the web browser. To import
+     *     this type, check out {@link Page}.
      *
      * * **Examples**:
      * ```typescript
@@ -467,7 +526,8 @@ export interface IScenario {
      *
      * * **Parameters**:
      *   * page: `Page` | the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
-     *     object, which provides APIs to interact with the web browser
+     *     object, which provides APIs to interact with the web browser. To import
+     *     this type, check out {@link Page}.
      *
      * * **Examples**:
      * ```typescript
@@ -496,7 +556,8 @@ export interface IScenario {
      *
      * * **Parameters**:
      *   * page: `Page` | the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
-     *     object, which provides APIs to interact with the web browser
+     *     object, which provides APIs to interact with the web browser. To import
+     *     this type, check out {@link Page}.
      *
      * * **Examples**:
      * ```typescript
@@ -539,7 +600,8 @@ export interface IScenario {
      *
      * * **Parameters**:
      *   * page: `Page` | the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
-     *     object, which provides APIs to interact with the web browser
+     *     object, which provides APIs to interact with the web browser. To import
+     *     this type, check out {@link Page}.
      *
      * * **Examples**:
      * ```typescript
@@ -582,7 +644,8 @@ export interface IScenario {
      *
      * * **Parameters**:
      *   * page: `Page` | the puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)
-     *     object, which provides APIs to interact with the web browser
+     *     object, which provides APIs to interact with the web browser. To import
+     *     this type, check out {@link Page}.
      * * **Returns**: a boolean value, if it returns `true`, memlab will consider
      *   the navigation completes, if it returns `false`, memlab will keep calling
      *   this callback until it returns `true`. This is an async callback, you can
@@ -766,6 +829,7 @@ export interface IDataBuilder {
  * Callback function to provide if the page is loaded.
  * For concrete example, check out {@link isPageLoaded}.
  * @param page - puppeteer's [Page](https://pptr.dev/api/puppeteer.page/) object.
+ * To import this type, check out {@link Page}.
  * @returns a boolean value, if it returns `true`, memlab will consider
  * the navigation completes, if it returns `false`, memlab will keep calling
  * this callback until it returns `true`. This is an async callback, you can
@@ -773,6 +837,8 @@ export interface IDataBuilder {
  */
 export declare type CheckPageLoadCallback = (page: Page) => Promise<boolean>;
 /** @internal */
+export declare type PageSetupCallback = (page: Page) => Promise<void>;
+/** @internal */
 export interface IE2EScenarioVisitPlan {
     name: string;
     appName: string;
@@ -786,6 +852,7 @@ export interface IE2EScenarioVisitPlan {
     dataBuilder: Optional<IDataBuilder>;
     isPageLoaded?: CheckPageLoadCallback;
     scenario?: IScenario;
+    pageSetup?: PageSetupCallback;
 }
 /** @internal */
 export declare type OperationArgs = {

package/dist/lib/Utils.d.ts

@@ -7,7 +7,7 @@
  * @format
  * @oncall web_perf_infra
  */
-import type { HaltOrThrowOptions } from './Types';
+import type { HaltOrThrowOptions, HeapNodeIdSet, ShellOptions } from './Types';
 import type { Browser, Page } from 'puppeteer';
 import type { AnyAyncFunction, AnyOptions, E2EStepInfo, IHeapSnapshot, IHeapNode, IHeapEdge, IScenario, ILeakFilter, LeakTracePathItem, RunMetaInfo, RawHeapSnapshot, Nullable, Optional } from './Types';
 declare function isHermesInternalObject(node: IHeapNode): boolean;
@@ -122,7 +122,11 @@ declare function getError(maybeError: unknown): Error;
 declare function isNodeDominatedByDeletionsArray(node: IHeapNode): boolean;
 declare function getUniqueID(): string;
 declare function getClosureSourceUrl(node: IHeapNode): Nullable<string>;
+export declare function runShell(command: string, options?: ShellOptions): string;
+export declare function getRetainedSize(node: IHeapNode): number;
+export declare function aggregateDominatorMetrics(ids: HeapNodeIdSet, snapshot: IHeapSnapshot, checkNodeCb: (node: IHeapNode) => boolean, nodeMetricsCb: (node: IHeapNode) => number): number;
 declare const _default: {
+    aggregateDominatorMetrics: typeof aggregateDominatorMetrics;
     applyToNodes: typeof applyToNodes;
     callAsync: typeof callAsync;
     camelCaseToReadableString: typeof camelCaseToReadableString;
@@ -148,6 +152,7 @@ declare const _default: {
     getReadableBytes: typeof getReadableBytes;
     getReadablePercent: typeof getReadablePercent;
     getReadableTime: typeof getReadableTime;
+    getRetainedSize: typeof getRetainedSize;
     getRunMetaFilePath: typeof getRunMetaFilePath;
     getScenarioName: typeof getScenarioName;
     getSingleSnapshotFileForAnalysis: typeof getSingleSnapshotFileForAnalysis;
@@ -215,6 +220,7 @@ declare const _default: {
     repeat: typeof repeat;
     resolveFilePath: typeof resolveFilePath;
     resolveSnapshotFilePath: typeof resolveSnapshotFilePath;
+    runShell: typeof runShell;
     setIsAlternateNode: typeof setIsAlternateNode;
     setIsRegularFiberNode: typeof setIsRegularFiberNode;
     shouldShowMoreInfo: typeof shouldShowMoreInfo;

package/dist/lib/Utils.js

@@ -44,9 +44,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.resolveSnapshotFilePath = void 0;
+exports.aggregateDominatorMetrics = exports.getRetainedSize = exports.runShell = exports.resolveSnapshotFilePath = void 0;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
+const child_process_1 = __importDefault(require("child_process"));
 const process_1 = __importDefault(require("process"));
 const Config_1 = __importStar(require("./Config"));
 const Console_1 = __importDefault(require("./Console"));
@@ -1751,7 +1752,52 @@ function getClosureSourceUrl(node) {
     const url = (_b = (_a = urlNode === null || urlNode === void 0 ? void 0 : urlNode.toStringNode()) === null || _a === void 0 ? void 0 : _a.stringValue) !== null && _b !== void 0 ? _b : null;
     return url;
 }
+function runShell(command, options = {}) {
+    var _a, _b, _c;
+    const runningDir = (_b = (_a = options.dir) !== null && _a !== void 0 ? _a : Config_1.default.workDir) !== null && _b !== void 0 ? _b : FileManager_1.default.getTmpDir();
+    const execOptions = {
+        cwd: runningDir,
+        stdio: options.disconnectStdio
+            ? []
+            : [process_1.default.stdin, process_1.default.stdout, process_1.default.stderr],
+    };
+    if (process_1.default.platform !== 'win32') {
+        execOptions.shell = '/bin/bash';
+    }
+    let ret = '';
+    try {
+        ret = child_process_1.default.execSync(command, execOptions);
+    }
+    catch (ex) {
+        if (Config_1.default.verbose) {
+            if (ex instanceof Error) {
+                Console_1.default.lowLevel(ex.message);
+                Console_1.default.lowLevel((_c = ex.stack) !== null && _c !== void 0 ? _c : '');
+            }
+        }
+        if (options.ignoreError === true) {
+            return '';
+        }
+        __1.utils.haltOrThrow(`Error when executing command: ${command}`);
+    }
+    return ret && ret.toString('UTF-8');
+}
+exports.runShell = runShell;
+function getRetainedSize(node) {
+    return node.retainedSize;
+}
+exports.getRetainedSize = getRetainedSize;
+function aggregateDominatorMetrics(ids, snapshot, checkNodeCb, nodeMetricsCb) {
+    let ret = 0;
+    const dominators = __1.utils.getConditionalDominatorIds(ids, snapshot, checkNodeCb);
+    __1.utils.applyToNodes(dominators, snapshot, node => {
+        ret += nodeMetricsCb(node);
+    });
+    return ret;
+}
+exports.aggregateDominatorMetrics = aggregateDominatorMetrics;
 exports.default = {
+    aggregateDominatorMetrics,
     applyToNodes,
     callAsync,
     camelCaseToReadableString,
@@ -1777,6 +1823,7 @@ exports.default = {
     getReadableBytes,
     getReadablePercent,
     getReadableTime,
+    getRetainedSize,
     getRunMetaFilePath,
     getScenarioName,
     getSingleSnapshotFileForAnalysis,
@@ -1844,6 +1891,7 @@ exports.default = {
     repeat,
     resolveFilePath,
     resolveSnapshotFilePath,
+    runShell,
     setIsAlternateNode,
     setIsRegularFiberNode,
     shouldShowMoreInfo,

package/dist/lib/heap-data/HeapNode.js

@@ -257,10 +257,9 @@ class HeapNode {
     get location() {
         const heapSnapshot = this.heapSnapshot;
         const locationIdx = heapSnapshot._nodeIdx2LocationIdx[this.idx];
-        if (locationIdx == null) {
-            return null;
-        }
-        return new HeapLocation_1.default(heapSnapshot, locationIdx);
+        return locationIdx == null
+            ? null
+            : new HeapLocation_1.default(heapSnapshot, locationIdx);
     }
     // search reference by edge name and edge type
     getReference(edgeName, edgeType) {

package/dist/lib/heap-data/HeapSnapshot.js

@@ -206,8 +206,8 @@ class HeapSnapshot {
         const locationFieldsCount = this._locationFieldsCount;
         let locationIdx = 0;
         while (locationIdx < this._locationCount) {
-            const id = locations[locationIdx * locationFieldsCount + this._locationObjectIndexOffset];
-            this._nodeIdx2LocationIdx[id] = locationIdx;
+            const nodeIndex = locations[locationIdx * locationFieldsCount + this._locationObjectIndexOffset];
+            this._nodeIdx2LocationIdx[nodeIndex] = locationIdx;
             ++locationIdx;
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memlab/core",
-  "version": "1.1.16",
+  "version": "1.1.17",
   "license": "MIT",
   "description": "memlab core libraries",
   "author": "Liang Gong <lgong@fb.com>",
@@ -59,7 +59,7 @@
   "scripts": {
     "build-pkg": "tsc",
     "test-pkg": "jest .",
-    "publish-patch": "npm version patch --force && npm publish",
+    "publish-patch": "npm publish",
     "clean-pkg": "rm -rf ./dist && rm -rf ./node_modules && rm -f ./tsconfig.tsbuildinfo"
   },
   "bugs": {