@memlab/core 1.1.14 → 1.1.17

package/dist/index.d.ts

@@ -50,5 +50,9 @@ export * from './lib/PackageInfoLoader';
 /** @internal */
 export { default as SequentialClustering } from './trace-cluster/SequentialClustering';
 /** @internal */
+export { default as MultiIterationSeqClustering } from './trace-cluster/MultiIterationSeqClustering';
+/** @internal */
 export { default as TraceFinder } from './paths/TraceFinder';
+/** @internal */
+export * from './trace-cluster/ClusterUtils';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -35,7 +35,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TraceFinder = exports.SequentialClustering = exports.EvaluationMetric = exports.NormalizedTrace = exports.leakClusterLogger = exports.ProcessManager = exports.modes = exports.constant = exports.analysis = exports.browserInfo = exports.serializer = exports.fileManager = exports.utils = exports.BaseOption = exports.info = exports.config = exports.registerPackage = void 0;
+exports.TraceFinder = exports.MultiIterationSeqClustering = exports.SequentialClustering = exports.EvaluationMetric = exports.NormalizedTrace = exports.leakClusterLogger = exports.ProcessManager = exports.modes = exports.constant = exports.analysis = exports.browserInfo = exports.serializer = exports.fileManager = exports.utils = exports.BaseOption = exports.info = exports.config = exports.registerPackage = void 0;
 const path_1 = __importDefault(require("path"));
 const PackageInfoLoader_1 = require("./lib/PackageInfoLoader");
 /** @internal */
@@ -101,5 +101,10 @@ __exportStar(require("./lib/PackageInfoLoader"), exports);
 var SequentialClustering_1 = require("./trace-cluster/SequentialClustering");
 Object.defineProperty(exports, "SequentialClustering", { enumerable: true, get: function () { return __importDefault(SequentialClustering_1).default; } });
 /** @internal */
+var MultiIterationSeqClustering_1 = require("./trace-cluster/MultiIterationSeqClustering");
+Object.defineProperty(exports, "MultiIterationSeqClustering", { enumerable: true, get: function () { return __importDefault(MultiIterationSeqClustering_1).default; } });
+/** @internal */
 var TraceFinder_1 = require("./paths/TraceFinder");
 Object.defineProperty(exports, "TraceFinder", { enumerable: true, get: function () { return __importDefault(TraceFinder_1).default; } });
+/** @internal */
+__exportStar(require("./trace-cluster/ClusterUtils"), exports);

package/dist/lib/Config.d.ts

@@ -71,6 +71,7 @@ export declare class MemLabConfig {
     userDataDir: string;
     curDataDir: string;
     webSourceDir: string;
+    debugDataDir: string;
     runMetaFile: string;
     snapshotSequenceFile: string;
     exploreResultFile: string;
@@ -90,6 +91,7 @@ export declare class MemLabConfig {
     traceClusterOutDir: string;
     traceJsonOutDir: string;
     metricsOutDir: string;
+    heapAnalysisLogDir: string;
     reportScreenshotFile: string;
     newUniqueClusterDir: string;
     staleUniqueClusterDir: string;
@@ -188,9 +190,13 @@ export declare class MemLabConfig {
     mlClusteringLinkageMaxDistance: number;
     mlMaxDF: number;
     isSequentialClustering: boolean;
+    isMultiIterationSeqClustering: boolean;
     seqClusteringSplitCount: number;
+    multiIterSeqClusteringIteration: number;
+    multiIterSeqClusteringSampleSize: number;
     seqClusteringIsRandomChunks: boolean;
     instrumentJS: boolean;
+    interceptScript: boolean;
     constructor(options?: ConfigOption);
     private initInternalConfigs;
     private init;

package/dist/lib/Config.js

@@ -116,7 +116,9 @@ class MemLabConfig {
         this.snapshotHasDetachedness = false;
         // by default running in regular mode
         this.runningMode = RunningModes_1.default.get('regular', this);
-        // intercept and rewrite JavaScript Code in browser
+        // intercept and log JavaScript Code in browser
+        this.interceptScript = false;
+        // rewrite JavaScript Code in browser
         this.instrumentJS = false;
         // external heap snapshot paths, if enabled
         this.externalSnapshotFilePaths = [];
@@ -139,8 +141,15 @@ class MemLabConfig {
         this.mlMaxDF = 1;
         // if true, evaluating results with sequential clustering
         this.isSequentialClustering = false;
+        // if true, evaluating results with sequential
+        // clustering with multiple iterations
+        this.isMultiIterationSeqClustering = false;
         // split the sample leak traces into 4 smaller ones by default.
         this.seqClusteringSplitCount = 4;
+        // the number of iterations for multi-iteration sequential clustering
+        this.multiIterSeqClusteringIteration = 1;
+        // the number of trace samples to retain from each cluster
+        this.multiIterSeqClusteringSampleSize = Infinity;
         // if true, split dataset into trunks
         // with random order for sequential clustering
         this.seqClusteringIsRandomChunks = false;

package/dist/lib/Console.d.ts

@@ -23,6 +23,7 @@ declare class MemLabConsole {
     private config;
     private sections;
     private log;
+    private logFileSet;
     private styles;
     private static singleton;
     protected constructor();
@@ -40,6 +41,8 @@ declare class MemLabConsole {
     private shouldBeConcise;
     private clearPrevOverwriteMsg;
     private printStr;
+    registerLogFile(logFile: string): void;
+    unregisterLogFile(logFile: string): void;
     beginSection(name: string): void;
     endSection(name: string): void;
     setConfig(config: MemLabConfig): void;

package/dist/lib/Console.js

@@ -14,10 +14,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const chalk_1 = __importDefault(require("chalk"));
 const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 const readline_1 = __importDefault(require("readline"));
 const string_width_1 = __importDefault(require("string-width"));
 const stdout = process.stdout;
 const TABLE_MAX_WIDTH = 50;
+const LOG_BUFFER_LENGTH = 100;
 const prevLine = '\x1b[F';
 const eraseLine = '\x1b[K';
 const barComplete = chalk_1.default.green('\u2588');
@@ -42,10 +44,21 @@ function formatTableArg(arg) {
         }
     });
 }
+function registerExitCleanup(inst, exitHandler) {
+    const p = process;
+    // normal exit
+    p.on('exit', exitHandler.bind(null, { cleanup: true }));
+    // ctrl + c event
+    p.on('SIGINT', exitHandler.bind(null, { exit: true }));
+    // kill pid
+    p.on('SIGUSR1', exitHandler.bind(null, { exit: true }));
+    p.on('SIGUSR2', exitHandler.bind(null, { exit: true }));
+}
 class MemLabConsole {
     constructor() {
         this.config = {};
         this.log = [];
+        this.logFileSet = new Set();
         this.styles = {
             top: (msg) => msg,
             high: chalk_1.default.dim.bind(chalk_1.default),
@@ -67,12 +80,15 @@ class MemLabConsole {
         }
         const inst = new MemLabConsole();
         MemLabConsole.singleton = inst;
-        // clean up output
+        const exitHandler = (
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        _options, 
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        process.on('exit', (_code) => {
-            inst.flushLog();
+        _exitCode) => {
+            inst.flushLog({ sync: true });
             inst.clearPrevOverwriteMsg();
-        });
+        };
+        registerExitCleanup(inst, exitHandler);
         return inst;
     }
     style(msg, name) {
@@ -98,28 +114,49 @@ class MemLabConsole {
     }
     logMsg(msg) {
         // remove control characters
-        const rawMsg = msg
+        const lines = msg.split('\n').map(line => line
             // eslint-disable-next-line no-control-regex
             .replace(/[\u0000-\u001F\u007F-\u009F]/g, '')
-            .replace(/\[\d{1,3}m/g, '');
-        this.log.push(rawMsg);
-        if (this.log.length > 20) {
-            this.flushLog();
+            .replace(/\[\d{1,3}m/g, ''));
+        this.log.push(...lines);
+        if (this.log.length > LOG_BUFFER_LENGTH) {
+            this.flushLog({ sync: true });
         }
     }
-    flushLog() {
+    flushLog(options = {}) {
         const str = this.log.join('\n');
-        if (str.length > 0) {
-            const file = this.config.consoleLogFile;
-            fs_1.default.appendFile(file, str + '\n', 'UTF-8', () => {
-                // NOOP
-            });
-        }
         this.log = [];
+        if (str.length === 0) {
+            return;
+        }
+        // synchronous logging
+        if (options.sync) {
+            for (const logFile of this.logFileSet) {
+                try {
+                    fs_1.default.appendFileSync(logFile, str + '\n', 'UTF-8');
+                }
+                catch (_a) {
+                    // fail silently
+                }
+            }
+        }
+        else {
+            // async logging
+            const emptyCallback = () => {
+                // no op
+            };
+            for (const logFile of this.logFileSet) {
+                try {
+                    fs_1.default.appendFile(logFile, str + '\n', 'UTF-8', emptyCallback);
+                }
+                catch (_b) {
+                    // fail silently
+                }
+            }
+        }
     }
     pushMsg(msg, options = {}) {
-        const len = this.sections.arr.length;
-        if (this.config.isContinuousTest || len === 0) {
+        if (this.sections.arr.length === 0) {
             return;
         }
         // calculate each line's visible width
@@ -140,7 +177,9 @@ class MemLabConsole {
         if (!section || section.msgs.length === 0) {
             return;
         }
-        stdout.write(eraseLine);
+        if (!this.config.muteConsole) {
+            stdout.write(eraseLine);
+        }
         const msg = section.msgs.pop();
         if (!msg) {
             return;
@@ -150,8 +189,10 @@ class MemLabConsole {
             const line = (_a = lines.pop()) !== null && _a !== void 0 ? _a : 0;
             const width = stdout.columns;
             let n = line === 0 ? 1 : Math.ceil(line / width);
-            while (n-- > 0) {
-                stdout.write(prevLine + eraseLine);
+            if (!this.config.muteConsole && !this.config.isTest) {
+                while (n-- > 0) {
+                    stdout.write(prevLine + eraseLine);
+                }
             }
         }
     }
@@ -189,11 +230,21 @@ class MemLabConsole {
         this.clearPrevMsgInLastSection();
     }
     printStr(msg, options = {}) {
-        if (this.config.isTest || this.config.muteConsole) {
+        this.pushMsg(msg, options);
+        if (this.config.isTest) {
             return;
         }
-        console.log(msg);
-        this.pushMsg(msg, options);
+        if (this.config.isContinuousTest || !this.config.muteConsole) {
+            console.log(msg);
+        }
+    }
+    registerLogFile(logFile) {
+        this.flushLog({ sync: true });
+        this.logFileSet.add(path_1.default.resolve(logFile));
+    }
+    unregisterLogFile(logFile) {
+        this.flushLog({ sync: true });
+        this.logFileSet.delete(path_1.default.resolve(logFile));
     }
     beginSection(name) {
         if (this.config.isContinuousTest) {
@@ -300,14 +351,16 @@ class MemLabConsole {
         this.printStr('');
     }
     overwrite(msg, options = {}) {
-        if (this.config.isTest || this.config.muteConsole) {
+        const str = this.style(msg, options.level || 'low');
+        if (this.config.isContinuousTest) {
+            this.printStr(msg, { isOverwrite: false });
             return;
         }
-        if (this.config.isContinuousTest) {
-            return console.log(msg);
+        if (this.config.isTest || this.config.muteConsole) {
+            this.printStr(str, { isOverwrite: true });
+            return;
         }
         this.clearPrevOverwriteMsg();
-        const str = this.style(msg, options.level || 'low');
         this.printStr(str, { isOverwrite: true });
     }
     waitForConsole(query) {

package/dist/lib/FileManager.d.ts

@@ -32,8 +32,11 @@ export declare class FileManager {
     getCurDataDir(options: FileOption): string;
     getWebSourceDir(options?: FileOption): string;
     getWebSourceMetaFile(options?: FileOption): string;
+    getDebugDataDir(options?: FileOption): string;
+    getDebugSourceFile(options?: FileOption): string;
     getPersistDataDir(options: FileOption): string;
     getLoggerOutDir(options?: FileOption): string;
+    getHeapAnalysisLogDir(options?: FileOption): string;
     getTraceClustersDir(options?: FileOption): string;
     getTraceJSONDir(options?: FileOption): string;
     getUnclassifiedTraceClusterDir(options?: FileOption): string;
@@ -61,6 +64,7 @@ export declare class FileManager {
         controlWorkDir: string;
         testWorkDir: string;
     };
+    initNewHeapAnalysisLogFile(options?: FileOption): string;
     getAndInitTSCompileIntermediateDir(): string;
     clearDataDirs(options?: FileOption): void;
     emptyDirIfExists(dir: string): void;

package/dist/lib/FileManager.js

@@ -98,12 +98,22 @@ class FileManager {
     getWebSourceMetaFile(options = {}) {
         return path_1.default.join(this.getWebSourceDir(options), 'files.json');
     }
+    getDebugDataDir(options = {}) {
+        return path_1.default.join(this.getDataBaseDir(options), 'debug');
+    }
+    getDebugSourceFile(options = {}) {
+        return path_1.default.join(this.getDebugDataDir(options), 'file.js');
+    }
     getPersistDataDir(options) {
         return path_1.default.join(this.getDataBaseDir(options), 'persist');
     }
     getLoggerOutDir(options = {}) {
         return path_1.default.join(this.getDataBaseDir(options), 'logger');
     }
+    // all heap analysis results generated
+    getHeapAnalysisLogDir(options = {}) {
+        return path_1.default.join(this.getLoggerOutDir(options), 'heap-analysis');
+    }
     // all trace clusters generated from the current run
     getTraceClustersDir(options = {}) {
         return path_1.default.join(this.getLoggerOutDir(options), 'trace-clusters');
@@ -200,6 +210,15 @@ class FileManager {
         const testWorkDir = joinAndProcessDir({}, expDir, 'test');
         return { controlWorkDir, testWorkDir };
     }
+    // create a unique log file created for heap analysis output
+    initNewHeapAnalysisLogFile(options = {}) {
+        const dir = this.getHeapAnalysisLogDir(options);
+        const file = path_1.default.join(dir, `analysis-${Utils_1.default.getUniqueID()}-out.log`);
+        if (!fs_extra_1.default.existsSync(file)) {
+            fs_extra_1.default.createFileSync(file);
+        }
+        return file;
+    }
     getAndInitTSCompileIntermediateDir() {
         const dir = path_1.default.join(this.getTmpDir(), 'memlab-code');
         this.rmDir(dir);
@@ -212,6 +231,7 @@ class FileManager {
             return;
         }
         this.emptyDirIfExists(this.getWebSourceDir(options));
+        this.emptyDirIfExists(this.getDebugDataDir(options));
         const dataSuffix = ['.heapsnapshot', '.json', '.png'];
         const files = fs_extra_1.default.readdirSync(curDataDir);
         for (const file of files) {
@@ -238,6 +258,8 @@ class FileManager {
         this.emptyDirIfExists(this.getUnclassifiedTraceClusterDir(options));
         // all unique cluster info
         this.emptyDirIfExists(this.getUniqueTraceClusterDir(options));
+        // all heap analysis results
+        this.emptyDirIfExists(this.getHeapAnalysisLogDir(options));
     }
     resetBrowserDir() {
         try {
@@ -294,9 +316,12 @@ class FileManager {
         const outDir = joinAndProcessDir(options, this.getDataOutDir(options));
         config.curDataDir = joinAndProcessDir(options, this.getCurDataDir(options));
         config.webSourceDir = joinAndProcessDir(options, this.getWebSourceDir(options));
+        config.debugDataDir = joinAndProcessDir(options, this.getDebugDataDir(options));
         config.dataBuilderDataDir = joinAndProcessDir(options, config.dataBaseDir, 'dataBuilder');
         config.persistentDataDir = joinAndProcessDir(options, this.getPersistDataDir(options));
+        // register the default log file
         config.consoleLogFile = path_1.default.join(config.curDataDir, 'console-log.txt');
+        Console_1.default.registerLogFile(config.consoleLogFile);
         config.runMetaFile = this.getRunMetaFile(options);
         config.snapshotSequenceFile = this.getSnapshotSequenceMetaFile(options);
         config.browserInfoSummary = path_1.default.join(config.curDataDir, 'browser-info.txt');
@@ -314,6 +339,8 @@ class FileManager {
         config.traceClusterOutDir = joinAndProcessDir(options, this.getTraceClustersDir(options));
         // detailed trace json files for visualization
         config.traceJsonOutDir = joinAndProcessDir(options, this.getTraceJSONDir(options));
+        // heap analysis results
+        config.heapAnalysisLogDir = joinAndProcessDir(options, this.getHeapAnalysisLogDir(options));
         config.metricsOutDir = joinAndProcessDir(options, loggerOutDir, 'metrics');
         config.reportScreenshotFile = path_1.default.join(outDir, 'report.png');
         const codeDataDir = this.getCodeDataDir();

package/dist/lib/HeapAnalyzer.d.ts

@@ -7,15 +7,10 @@
  * @format
  * @oncall web_perf_infra
  */
-import type { E2EStepInfo, HeapNodeIdSet, IHeapNode, IHeapSnapshot, IMemoryAnalystOptions, IMemoryAnalystSnapshotDiff, LeakTracePathItem, Optional, IOveralLeakInfo, TraceCluster, ISerializedInfo } from './Types';
+import type { E2EStepInfo, HeapNodeIdSet, IHeapSnapshot, IMemoryAnalystOptions, IMemoryAnalystSnapshotDiff, IOveralHeapInfo, LeakTracePathItem, Optional, IOveralLeakInfo, TraceCluster, ISerializedInfo } from './Types';
 import TraceFinder from '../paths/TraceFinder';
 declare class MemoryAnalyst {
     checkLeak(): Promise<ISerializedInfo[]>;
-    checkUnbound(options?: IMemoryAnalystOptions): Promise<void>;
-    breakDownMemoryByShapes(options?: {
-        file?: string;
-    }): Promise<void>;
-    detectUnboundGrowth(options?: IMemoryAnalystOptions): Promise<void>;
     detectMemoryLeaks(): Promise<ISerializedInfo[]>;
     visualizeMemoryUsage(options?: IMemoryAnalystOptions): void;
     focus(options?: {
@@ -23,19 +18,15 @@ declare class MemoryAnalyst {
     }): Promise<void>;
     shouldLoadCompleteSnapshot(tabsOrder: E2EStepInfo[], tab: E2EStepInfo): boolean;
     diffSnapshots(loadAll?: boolean): Promise<IMemoryAnalystSnapshotDiff>;
-    private calculateRetainedSizes;
     preparePathFinder(snapshot: IHeapSnapshot): TraceFinder;
     private dumpPageInteractionSummary;
     private dumpLeakSummaryToConsole;
     private filterLeakedObjects;
-    aggregateDominatorMetrics(ids: HeapNodeIdSet, snapshot: IHeapSnapshot, checkNodeCb: (node: IHeapNode) => boolean, nodeMetricsCb: (node: IHeapNode) => number): number;
-    private getOverallHeapInfo;
-    getRetainedSize(node: IHeapNode): number;
+    getOverallHeapInfo(snapshot: IHeapSnapshot, options?: {
+        force?: boolean;
+    }): Optional<IOveralHeapInfo>;
     getOverallLeakInfo(leakedNodeIds: HeapNodeIdSet, snapshot: IHeapSnapshot): Optional<IOveralLeakInfo>;
-    private printHeapInfo;
-    private breakDownSnapshotByShapes;
-    private isTrivialEdgeForBreakDown;
-    private breakDownByReferrers;
+    printHeapInfo(leakInfo: IOveralHeapInfo): void;
     private printHeapAndLeakInfo;
     private logLeakTraceSummary;
     searchLeakedTraces(leakedNodeIds: HeapNodeIdSet, snapshot: IHeapSnapshot): Promise<{