@memlab/core 1.1.4 → 1.1.7

package/dist/lib/heap-data/HeapSnapshot.js

@@ -18,11 +18,13 @@ const Console_1 = __importDefault(require("../Console"));
 const HeapNode_1 = __importDefault(require("./HeapNode"));
 const HeapEdge_1 = __importDefault(require("./HeapEdge"));
 const HeapUtils_1 = require("./HeapUtils");
+const MemLabTagStore_1 = __importDefault(require("./MemLabTagStore"));
 const EMPTY_UINT8_ARRAY = new Uint8Array(0);
 const EMPTY_UINT32_ARRAY = new Uint32Array(0);
 class HeapSnapshot {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     constructor(snapshot, _options = {}) {
+        this.isProcessed = false;
         this._nodeCount = -1;
         this._edgeCount = -1;
         this._nodeId2NodeIdx = {};
@@ -158,36 +160,7 @@ class HeapSnapshot {
         return detected;
     }
     hasObjectWithTag(tag) {
-        // get tagStore
-        let tagStore = null;
-        this.nodes.forEach((node) => {
-            if (node.name === 'MemLabTaggedStore' && node.type === 'object') {
-                tagStore = node;
-                return false;
-            }
-        });
-        if (tagStore == null) {
-            return false;
-        }
-        const store = tagStore;
-        // get tagStore.taggedObjects
-        const taggedObjects = store.getReferenceNode('taggedObjects', 'property');
-        if (taggedObjects == null) {
-            return false;
-        }
-        // get taggedObjects[tag]
-        const weakSet = taggedObjects.getReferenceNode(tag, 'property');
-        if (weakSet == null) {
-            return false;
-        }
-        // get weakSet.table
-        const table = weakSet.getReferenceNode('table');
-        if (table == null) {
-            return false;
-        }
-        // check if the table has any weak reference to any object
-        const ref = table.findAnyReference((edge) => edge.type === 'weak' && edge.toNode.name !== 'system / Oddball');
-        return ref != null;
+        return MemLabTagStore_1.default.hasObjectWithTag(this, tag);
     }
     getNodeById(id) {
         if (!(id in this._nodeId2NodeIdx)) {

package/dist/lib/heap-data/HeapStringNode.js

@@ -35,6 +35,8 @@ class HeapStringNode extends HeapNode_1.default {
             if (parentNode == null) {
                 throw (0, HeapUtils_1.throwError)(new Error('broken sliced string'));
             }
+            // sliced string in heap snapshot doesn't include
+            // the start index and the end index, so this may be inaccurate
             return parentNode.stringValue;
         }
         return this.name;

package/dist/lib/heap-data/MemLabTagStore.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+import type { AnyValue, IHeapSnapshot } from '../Types';
+declare type AnyObject = Record<AnyValue, AnyValue>;
+/** @internal */
+export default class MemLabTaggedStore {
+    taggedObjects: Record<string, WeakSet<AnyObject>>;
+    private constructor();
+    private static instance;
+    readonly id: string;
+    static getInstance(): MemLabTaggedStore;
+    static tagObject<T>(o: T, tag: string): void;
+    static hasObjectWithTag(heap: IHeapSnapshot, tag: string): boolean;
+}
+export {};
+//# sourceMappingURL=MemLabTagStore.d.ts.map

package/dist/lib/heap-data/MemLabTagStore.js

@@ -0,0 +1,110 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const __1 = require("../..");
+let uindex = 1;
+function getUniqueID() {
+    const randId = `${Math.random()}`;
+    return `${process.pid}-${Date.now()}-${randId}-${uindex++}`;
+}
+/** @internal */
+class MemLabTaggedStore {
+    constructor() {
+        this.id = getUniqueID();
+        this.taggedObjects = Object.create(null);
+    }
+    // make sure it's a singleton
+    static getInstance() {
+        if (!MemLabTaggedStore.instance) {
+            MemLabTaggedStore.instance = new MemLabTaggedStore();
+        }
+        return MemLabTaggedStore.instance;
+    }
+    // tag an object with a mark
+    static tagObject(o, tag) {
+        const store = MemLabTaggedStore.getInstance();
+        if (!store.taggedObjects[tag]) {
+            store.taggedObjects[tag] = new WeakSet();
+        }
+        store.taggedObjects[tag].add(o);
+    }
+    // check if any object in the heap snapshot has the mark
+    // tagged by this MemLabTaggedStore in this execution context
+    static hasObjectWithTag(heap, tag) {
+        const curContextTagStoreID = MemLabTaggedStore.getInstance().id;
+        let tagStore = null;
+        // get all MemLabTaggedStore instances in the heap snapshot
+        const stores = [];
+        heap.nodes.forEach((node) => {
+            if (node.name === 'MemLabTaggedStore' && node.type === 'object') {
+                stores.push(node);
+            }
+        });
+        // if no tag store found
+        if (stores.length === 0) {
+            return false;
+            // if there is only one store found
+        }
+        else if (stores.length === 1) {
+            tagStore = stores[0];
+            // if there are multiple MemLabTagStore instances
+            // found in the heap snapshot
+        }
+        else if (stores.length > 1) {
+            stores.forEach((node) => {
+                // in case multiple instances of MemLabTaggedStore exists
+                // in the heap snapshot, we need to make sure that the
+                // tag store is the one matching the current execution context
+                let storeID = '';
+                // match tag store id
+                node.forEachReference(edge => {
+                    var _a, _b;
+                    if (edge.name_or_index === 'id' && edge.toNode.isString) {
+                        storeID = (_b = (_a = edge.toNode.toStringNode()) === null || _a === void 0 ? void 0 : _a.stringValue) !== null && _b !== void 0 ? _b : '';
+                        return { stop: true };
+                    }
+                });
+                if (curContextTagStoreID === storeID) {
+                    tagStore = node;
+                }
+            });
+            if (tagStore == null) {
+                throw __1.utils.haltOrThrow('Multiple MemLabTagStore instances found in heap snapshot ' +
+                    'when checking object tags, please make sure only one memlab ' +
+                    'instance is running at a time and double check that memlab is ' +
+                    'not running in Jest concurrent mode.');
+            }
+        }
+        if (tagStore == null) {
+            return false;
+        }
+        const store = tagStore;
+        // get tagStore.taggedObjects
+        const taggedObjects = store.getReferenceNode('taggedObjects', 'property');
+        if (taggedObjects == null) {
+            return false;
+        }
+        // get taggedObjects[tag]
+        const weakSet = taggedObjects.getReferenceNode(tag, 'property');
+        if (weakSet == null) {
+            return false;
+        }
+        // get weakSet.table
+        const table = weakSet.getReferenceNode('table');
+        if (table == null) {
+            return false;
+        }
+        // check if the table has any weak reference to any object
+        const ref = table.findAnyReference((edge) => edge.type === 'weak' && edge.toNode.name !== 'system / Oddball');
+        return ref != null;
+    }
+}
+exports.default = MemLabTaggedStore;

package/dist/trace-cluster/TraceBucket.js

@@ -20,6 +20,7 @@ const Utils_1 = __importDefault(require("../lib/Utils"));
 const TraceElement_1 = require("./TraceElement");
 const TraceSimilarityStrategy_1 = __importDefault(require("./strategies/TraceSimilarityStrategy"));
 const TraceAsClusterStrategy_1 = __importDefault(require("./strategies/TraceAsClusterStrategy"));
+const MLTraceSimilarityStrategy_1 = __importDefault(require("./strategies/MLTraceSimilarityStrategy"));
 // sync up with html/intern/js/webspeed/memlab/lib/LeakCluster.js
 class NormalizedTrace {
     constructor(p = null, snapshot = null) {
@@ -157,7 +158,11 @@ class NormalizedTrace {
         };
     }
     static clusterLeakTraces(leakTraces) {
-        const { allClusters } = NormalizedTrace.diffTraces(leakTraces, []);
+        const { allClusters } = NormalizedTrace.diffTraces(leakTraces, [], {
+            strategy: Config_1.default.isMLClustering
+                ? new MLTraceSimilarityStrategy_1.default()
+                : undefined,
+        });
         const lastNodeFromTrace = (trace) => trace[trace.length - 1];
         const labaledLeakTraces = allClusters.reduce((acc, bucket) => {
             const lastNodeFromFirstTrace = lastNodeFromTrace(bucket[0]);

package/dist/trace-cluster/strategies/MLTraceSimilarityStrategy.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+import type { IClusterStrategy, LeakTrace, TraceDiff } from '../../lib/Types';
+export default class MLTraceSimilarityStrategy implements IClusterStrategy {
+    diffTraces(newLeakTraces: LeakTrace[]): TraceDiff;
+    traceToDoc(trace: LeakTrace): string;
+}
+//# sourceMappingURL=MLTraceSimilarityStrategy.d.ts.map

package/dist/trace-cluster/strategies/MLTraceSimilarityStrategy.js

@@ -0,0 +1,61 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Config_1 = __importDefault(require("../../lib/Config"));
+const DistanceMatrix_1 = require("./machine-learning/DistanceMatrix");
+const HAC_1 = require("./machine-learning/HAC");
+const TfidfVectorizer_1 = require("./machine-learning/TfidfVectorizer");
+class MLTraceSimilarityStrategy {
+    diffTraces(newLeakTraces) {
+        var _a;
+        const rawDocuments = newLeakTraces.map(this.traceToDoc);
+        const vectorizer = new TfidfVectorizer_1.TfidfVectorizer({ rawDocuments });
+        const tfidfs = vectorizer.computeTfidfs();
+        const dmatrix = (0, DistanceMatrix_1.distance)(tfidfs);
+        const result = (0, HAC_1.cluster)(rawDocuments.length, dmatrix, Config_1.default.mlClusteringLinkageMaxDistance);
+        const map = new Map();
+        for (let i = 0; i < result.length; i++) {
+            const traceIdx = result[i];
+            const repTrace = newLeakTraces[traceIdx];
+            const trace = newLeakTraces[i];
+            if (!map.has(repTrace)) {
+                map.set(repTrace, [repTrace]);
+            }
+            // to please linter
+            (_a = map.get(repTrace)) === null || _a === void 0 ? void 0 : _a.push(trace);
+        }
+        return {
+            allClusters: Array.from(map.values()),
+            staleClusters: [],
+            clustersToAdd: [],
+        };
+    }
+    traceToDoc(trace) {
+        const res = [];
+        for (const t of trace) {
+            let name = t.kind === 'node' ? String(t.name) : String(t.name_or_index);
+            if (name === '') {
+                name = '_null_';
+            }
+            name = name.replace(/ /g, '_');
+            name = name.replace(/\d/g, '');
+            if (name === '') {
+                name = '_number_';
+            }
+            res.push(name);
+        }
+        return res.join(' ');
+    }
+}
+exports.default = MLTraceSimilarityStrategy;

package/dist/trace-cluster/strategies/machine-learning/DistanceMatrix.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+export declare const distance: (tfidfs: Record<string, number>[]) => Float32Array;
+//# sourceMappingURL=DistanceMatrix.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/DistanceMatrix.js

@@ -0,0 +1,54 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.distance = void 0;
+const cache = new Map();
+const buildIntersection = (tfidfs, i, j) => {
+    const intersection = [];
+    if (!cache.has(i)) {
+        cache.set(i, Object.keys(tfidfs[i]));
+    }
+    if (!cache.has(j)) {
+        cache.set(j, Object.keys(tfidfs[j]));
+    }
+    const [keys, tfidf] = cache.get(i).length > cache.get(j).length
+        ? [cache.get(j), tfidfs[i]]
+        : [cache.get(i), tfidfs[j]];
+    for (const k of keys) {
+        if (tfidf[k]) {
+            intersection.push(k);
+        }
+    }
+    return intersection;
+};
+const distance = (tfidfs) => {
+    const n = tfidfs.length;
+    const distances = new Float32Array((n * (n - 1)) / 2);
+    let distIdx = 0;
+    const dotProducs = tfidfs.map(atfidf => Object.values(atfidf).reduce((sum, v) => sum + v * v, 0));
+    for (let i = 0; i < tfidfs.length; i++) {
+        const a = tfidfs[i];
+        for (let j = i + 1; j < tfidfs.length; j++) {
+            const b = tfidfs[j];
+            const intersection = buildIntersection(tfidfs, i, j);
+            const dotProdOfCommons = intersection.reduce((sum, vidx) => sum + a[vidx] * b[vidx], 0);
+            // TODO make it pluggable to use other distance measures like euclidean, manhattan
+            const cosineSimilarity = 1 -
+                dotProdOfCommons /
+                    (Math.sqrt(dotProducs[i]) / Math.sqrt(dotProducs[j]));
+            distances[distIdx] = cosineSimilarity;
+            distIdx++;
+        }
+    }
+    cache.clear();
+    return distances;
+};
+exports.distance = distance;

package/dist/trace-cluster/strategies/machine-learning/HAC.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+/**
+ *
+ * @param {*} nDocs number of docs
+ * @param {*} D condenced distance matrix
+ * @returns labels - list of doc ids as clusters
+ */
+export declare const cluster: (nDocs: number, condensedDistanceMatrix: Float32Array, maxDistanceThreshold: number) => number[] | Uint32Array;
+//# sourceMappingURL=HAC.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/HAC.js

@@ -0,0 +1,122 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cluster = void 0;
+const condensedIndex = (n, i, j) => {
+    if (i > j) {
+        return condensedIndex(n, j, i);
+    }
+    // to get distance between (i, j) think of this sequence.
+    // (n - 1) + (n - 2) + ... + (n - i) + (j - i) - 1
+    return n * i - (i * (i + 1)) / 2 + (j - i - 1);
+};
+function getRootLabel(array, idx) {
+    let rootIdx = idx;
+    while (array[rootIdx] !== rootIdx) {
+        rootIdx = array[rootIdx];
+    }
+    return rootIdx;
+}
+/**
+ *
+ * @param {*} nDocs number of docs
+ * @param {*} D condenced distance matrix
+ * @returns labels - list of doc ids as clusters
+ */
+const cluster = (nDocs, condensedDistanceMatrix, maxDistanceThreshold) => {
+    if (nDocs <= 1)
+        return [0];
+    const condencedDistanceMatrixCopy = new Float32Array(condensedDistanceMatrix);
+    const sizeOfClusters = new Uint32Array(nDocs).fill(1);
+    let chainLength = 0;
+    let clusterChain = [];
+    let traceAIdx = -1;
+    let traceBIdx = -1;
+    let currentMin = Number.MAX_SAFE_INTEGER;
+    let distanceBetweenTraces;
+    const labels = new Uint32Array(nDocs).map((_, idx) => idx);
+    for (let k = 0; k < nDocs - 1; k++) {
+        traceBIdx = -1;
+        if (chainLength === 0) {
+            for (let i = 0; i < nDocs; i++) {
+                if (sizeOfClusters[i] > 0) {
+                    clusterChain[0] = i;
+                    chainLength = 1;
+                    break;
+                }
+            }
+        }
+        while (chainLength > 0) {
+            traceAIdx = clusterChain[chainLength - 1];
+            if (chainLength > 1) {
+                traceBIdx = clusterChain[chainLength - 2];
+                currentMin =
+                    condencedDistanceMatrixCopy[condensedIndex(nDocs, traceAIdx, traceBIdx)];
+            }
+            else {
+                currentMin = Number.MAX_SAFE_INTEGER;
+            }
+            for (let i = 0; i < nDocs; i++) {
+                if (sizeOfClusters[i] == 0 || traceAIdx == i) {
+                    continue;
+                }
+                distanceBetweenTraces =
+                    condencedDistanceMatrixCopy[condensedIndex(nDocs, traceAIdx, i)];
+                if (distanceBetweenTraces < currentMin) {
+                    currentMin = distanceBetweenTraces;
+                    traceBIdx = i;
+                }
+            }
+            // make sure that traceA and traceB are closest to each other
+            if (chainLength > 1 &&
+                traceBIdx !== -1 &&
+                traceBIdx === clusterChain[chainLength - 2]) {
+                break;
+            }
+            clusterChain[chainLength] = traceBIdx;
+            chainLength = chainLength + 1;
+        }
+        clusterChain = [];
+        chainLength = 0;
+        if (currentMin > maxDistanceThreshold) {
+            sizeOfClusters[traceAIdx] = 0;
+            sizeOfClusters[traceBIdx] = 0;
+            continue;
+        }
+        if (traceAIdx === -1 || traceBIdx === -1) {
+            continue;
+        }
+        if (traceAIdx > traceBIdx) {
+            [traceAIdx, traceBIdx] = [traceBIdx, traceAIdx];
+        }
+        const nx = sizeOfClusters[traceAIdx];
+        const ny = sizeOfClusters[traceBIdx];
+        labels[traceAIdx] = traceBIdx;
+        sizeOfClusters[traceAIdx] = 0;
+        sizeOfClusters[traceBIdx] = nx + ny;
+        for (let i = 0; i < nDocs; i++) {
+            const ni = sizeOfClusters[i];
+            if (ni === 0 || i === traceBIdx) {
+                continue;
+            }
+            const d_xi = condencedDistanceMatrixCopy[condensedIndex(nDocs, i, traceAIdx)];
+            const d_yi = condencedDistanceMatrixCopy[condensedIndex(nDocs, i, traceBIdx)];
+            const size_x = nx;
+            const size_y = ny;
+            // TODO make it generic to support other linkage methods like complete, weighted etc...
+            const updatedDist = (size_x * d_xi + size_y * d_yi) / (size_x + size_y);
+            condencedDistanceMatrixCopy[condensedIndex(nDocs, i, traceBIdx)] =
+                updatedDist;
+        }
+    }
+    return labels.map((_, idx) => getRootLabel(labels, idx));
+};
+exports.cluster = cluster;

package/dist/trace-cluster/strategies/machine-learning/Ngram.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+export declare function nGram(n: number, terms: string[]): string[];
+//# sourceMappingURL=Ngram.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/Ngram.js

@@ -0,0 +1,22 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nGram = void 0;
+function nGram(n, terms) {
+    const nGrams = [];
+    let index = 0;
+    while (index <= terms.length - n) {
+        nGrams[index] = terms.slice(index, index + n).join(' ');
+        ++index;
+    }
+    return nGrams;
+}
+exports.nGram = nGram;

package/dist/trace-cluster/strategies/machine-learning/TfidfVectorizer.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @emails oncall+ws_labs
+ * @format
+ */
+interface TfidfVectorizerProps {
+    rawDocuments: string[];
+    maxDF?: number;
+}
+export declare class TfidfVectorizer {
+    rawDocuments: string[];
+    vocabulary: Record<string, string>;
+    documentFrequency: Record<string, number>;
+    maxDF: number;
+    documents: Record<string, number>[];
+    tfidfs: Record<string, number>[];
+    constructor({ rawDocuments, maxDF }: TfidfVectorizerProps);
+    computeTfidfs(): Record<string, number>[];
+    tokenize(text: string): string[];
+    buildVocabulary(tokenizedDocuments: string[][]): Record<string, string>;
+    processDocuments(tokenizedDocuments: string[][]): void;
+    limit(): void;
+    /**
+     * Smooth idf weights by adding 1 to document frequencies (DF), as if an extra
+     * document was seen containing every term in the collection exactly once.
+     * This prevents zero divisions.
+     * */
+    smooth(): void;
+    buildTfidfs(): Record<string, number>[];
+    tf(vocabIdx: string, document: Record<string, number>): number;
+    idf(vocabIdx: string): number;
+}
+export {};
+//# sourceMappingURL=TfidfVectorizer.d.ts.map