@memlab/core 1.1.5 → 1.1.9

package/dist/trace-cluster/strategies/machine-learning/DistanceMatrix.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+export declare const distance: (tfidfs: Record<string, number>[]) => Float32Array;
+//# sourceMappingURL=DistanceMatrix.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/DistanceMatrix.js

@@ -0,0 +1,54 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.distance = void 0;
+const cache = new Map();
+const buildIntersection = (tfidfs, i, j) => {
+    const intersection = [];
+    if (!cache.has(i)) {
+        cache.set(i, Object.keys(tfidfs[i]));
+    }
+    if (!cache.has(j)) {
+        cache.set(j, Object.keys(tfidfs[j]));
+    }
+    const [keys, tfidf] = cache.get(i).length > cache.get(j).length
+        ? [cache.get(j), tfidfs[i]]
+        : [cache.get(i), tfidfs[j]];
+    for (const k of keys) {
+        if (tfidf[k]) {
+            intersection.push(k);
+        }
+    }
+    return intersection;
+};
+const distance = (tfidfs) => {
+    const n = tfidfs.length;
+    const distances = new Float32Array((n * (n - 1)) / 2);
+    let distIdx = 0;
+    const dotProducs = tfidfs.map(atfidf => Object.values(atfidf).reduce((sum, v) => sum + v * v, 0));
+    for (let i = 0; i < tfidfs.length; i++) {
+        const a = tfidfs[i];
+        for (let j = i + 1; j < tfidfs.length; j++) {
+            const b = tfidfs[j];
+            const intersection = buildIntersection(tfidfs, i, j);
+            const dotProdOfCommons = intersection.reduce((sum, vidx) => sum + a[vidx] * b[vidx], 0);
+            // TODO make it pluggable to use other distance measures like euclidean, manhattan
+            const cosineSimilarity = 1 -
+                dotProdOfCommons /
+                    (Math.sqrt(dotProducs[i]) / Math.sqrt(dotProducs[j]));
+            distances[distIdx] = cosineSimilarity;
+            distIdx++;
+        }
+    }
+    cache.clear();
+    return distances;
+};
+exports.distance = distance;

package/dist/trace-cluster/strategies/machine-learning/HAC.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+/**
+ *
+ * @param {*} nDocs number of docs
+ * @param {*} D condenced distance matrix
+ * @returns labels - list of doc ids as clusters
+ */
+export declare const cluster: (nDocs: number, condensedDistanceMatrix: Float32Array, maxDistanceThreshold: number) => number[] | Uint32Array;
+//# sourceMappingURL=HAC.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/HAC.js

@@ -0,0 +1,122 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cluster = void 0;
+const condensedIndex = (n, i, j) => {
+    if (i > j) {
+        return condensedIndex(n, j, i);
+    }
+    // to get distance between (i, j) think of this sequence.
+    // (n - 1) + (n - 2) + ... + (n - i) + (j - i) - 1
+    return n * i - (i * (i + 1)) / 2 + (j - i - 1);
+};
+function getRootLabel(array, idx) {
+    let rootIdx = idx;
+    while (array[rootIdx] !== rootIdx) {
+        rootIdx = array[rootIdx];
+    }
+    return rootIdx;
+}
+/**
+ *
+ * @param {*} nDocs number of docs
+ * @param {*} D condenced distance matrix
+ * @returns labels - list of doc ids as clusters
+ */
+const cluster = (nDocs, condensedDistanceMatrix, maxDistanceThreshold) => {
+    if (nDocs <= 1)
+        return [0];
+    const condencedDistanceMatrixCopy = new Float32Array(condensedDistanceMatrix);
+    const sizeOfClusters = new Uint32Array(nDocs).fill(1);
+    let chainLength = 0;
+    let clusterChain = [];
+    let traceAIdx = -1;
+    let traceBIdx = -1;
+    let currentMin = Number.MAX_SAFE_INTEGER;
+    let distanceBetweenTraces;
+    const labels = new Uint32Array(nDocs).map((_, idx) => idx);
+    for (let k = 0; k < nDocs - 1; k++) {
+        traceBIdx = -1;
+        if (chainLength === 0) {
+            for (let i = 0; i < nDocs; i++) {
+                if (sizeOfClusters[i] > 0) {
+                    clusterChain[0] = i;
+                    chainLength = 1;
+                    break;
+                }
+            }
+        }
+        while (chainLength > 0) {
+            traceAIdx = clusterChain[chainLength - 1];
+            if (chainLength > 1) {
+                traceBIdx = clusterChain[chainLength - 2];
+                currentMin =
+                    condencedDistanceMatrixCopy[condensedIndex(nDocs, traceAIdx, traceBIdx)];
+            }
+            else {
+                currentMin = Number.MAX_SAFE_INTEGER;
+            }
+            for (let i = 0; i < nDocs; i++) {
+                if (sizeOfClusters[i] == 0 || traceAIdx == i) {
+                    continue;
+                }
+                distanceBetweenTraces =
+                    condencedDistanceMatrixCopy[condensedIndex(nDocs, traceAIdx, i)];
+                if (distanceBetweenTraces < currentMin) {
+                    currentMin = distanceBetweenTraces;
+                    traceBIdx = i;
+                }
+            }
+            // make sure that traceA and traceB are closest to each other
+            if (chainLength > 1 &&
+                traceBIdx !== -1 &&
+                traceBIdx === clusterChain[chainLength - 2]) {
+                break;
+            }
+            clusterChain[chainLength] = traceBIdx;
+            chainLength = chainLength + 1;
+        }
+        clusterChain = [];
+        chainLength = 0;
+        if (currentMin > maxDistanceThreshold) {
+            sizeOfClusters[traceAIdx] = 0;
+            sizeOfClusters[traceBIdx] = 0;
+            continue;
+        }
+        if (traceAIdx === -1 || traceBIdx === -1) {
+            continue;
+        }
+        if (traceAIdx > traceBIdx) {
+            [traceAIdx, traceBIdx] = [traceBIdx, traceAIdx];
+        }
+        const nx = sizeOfClusters[traceAIdx];
+        const ny = sizeOfClusters[traceBIdx];
+        labels[traceAIdx] = traceBIdx;
+        sizeOfClusters[traceAIdx] = 0;
+        sizeOfClusters[traceBIdx] = nx + ny;
+        for (let i = 0; i < nDocs; i++) {
+            const ni = sizeOfClusters[i];
+            if (ni === 0 || i === traceBIdx) {
+                continue;
+            }
+            const d_xi = condencedDistanceMatrixCopy[condensedIndex(nDocs, i, traceAIdx)];
+            const d_yi = condencedDistanceMatrixCopy[condensedIndex(nDocs, i, traceBIdx)];
+            const size_x = nx;
+            const size_y = ny;
+            // TODO make it generic to support other linkage methods like complete, weighted etc...
+            const updatedDist = (size_x * d_xi + size_y * d_yi) / (size_x + size_y);
+            condencedDistanceMatrixCopy[condensedIndex(nDocs, i, traceBIdx)] =
+                updatedDist;
+        }
+    }
+    return labels.map((_, idx) => getRootLabel(labels, idx));
+};
+exports.cluster = cluster;

package/dist/trace-cluster/strategies/machine-learning/Ngram.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+export declare function nGram(n: number, terms: string[]): string[];
+//# sourceMappingURL=Ngram.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/Ngram.js

@@ -0,0 +1,22 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nGram = void 0;
+function nGram(n, terms) {
+    const nGrams = [];
+    let index = 0;
+    while (index <= terms.length - n) {
+        nGrams[index] = terms.slice(index, index + n).join(' ');
+        ++index;
+    }
+    return nGrams;
+}
+exports.nGram = nGram;

package/dist/trace-cluster/strategies/machine-learning/TfidfVectorizer.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+interface TfidfVectorizerProps {
+    rawDocuments: string[];
+    maxDF?: number;
+}
+export declare class TfidfVectorizer {
+    rawDocuments: string[];
+    vocabulary: Record<string, string>;
+    documentFrequency: Record<string, number>;
+    maxDF: number;
+    documents: Record<string, number>[];
+    tfidfs: Record<string, number>[];
+    constructor({ rawDocuments, maxDF }: TfidfVectorizerProps);
+    computeTfidfs(): Record<string, number>[];
+    tokenize(text: string): string[];
+    buildVocabulary(tokenizedDocuments: string[][]): Record<string, string>;
+    processDocuments(tokenizedDocuments: string[][]): void;
+    limit(): void;
+    /**
+     * Smooth idf weights by adding 1 to document frequencies (DF), as if an extra
+     * document was seen containing every term in the collection exactly once.
+     * This prevents zero divisions.
+     * */
+    smooth(): void;
+    buildTfidfs(): Record<string, number>[];
+    tf(vocabIdx: string, document: Record<string, number>): number;
+    idf(vocabIdx: string): number;
+}
+export {};
+//# sourceMappingURL=TfidfVectorizer.d.ts.map

package/dist/trace-cluster/strategies/machine-learning/TfidfVectorizer.js

@@ -0,0 +1,144 @@
+"use strict";
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @oncall ws_labs
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TfidfVectorizer = void 0;
+const Config_1 = __importDefault(require("../../../lib/Config"));
+const Ngram_1 = require("./Ngram");
+const SMOOTHING_KEY = '__smoothObjectKey';
+const VOCAB_IDX_FOR_DOC_WITH_HIGH_DF = '-1';
+class TfidfVectorizer {
+    constructor({ rawDocuments, maxDF }) {
+        this.rawDocuments = [];
+        this.vocabulary = Object.create(null);
+        this.documentFrequency = Object.create(null);
+        this.documents = [];
+        this.rawDocuments = rawDocuments;
+        this.maxDF = maxDF !== null && maxDF !== void 0 ? maxDF : Config_1.default.mlMaxDF;
+    }
+    computeTfidfs() {
+        const tokenizedDocuments = this.rawDocuments.map(this.tokenize);
+        this.vocabulary = this.buildVocabulary(tokenizedDocuments);
+        this.processDocuments(tokenizedDocuments);
+        this.limit();
+        this.smooth();
+        this.tfidfs = this.buildTfidfs();
+        return this.tfidfs;
+    }
+    tokenize(text) {
+        const terms = text.split(' ');
+        return [...terms, ...(0, Ngram_1.nGram)(2, terms), ...(0, Ngram_1.nGram)(3, terms)];
+    }
+    buildVocabulary(tokenizedDocuments) {
+        let vocabIdx = 0;
+        const vocabulary = Object.create(null);
+        tokenizedDocuments.forEach(doc => {
+            doc.forEach(term => {
+                if (!vocabulary[String(term)]) {
+                    vocabulary[String(term)] = String(vocabIdx);
+                    vocabIdx++;
+                }
+            });
+        });
+        return vocabulary;
+    }
+    processDocuments(tokenizedDocuments) {
+        tokenizedDocuments.forEach(terms => {
+            const document = {};
+            terms.forEach(t => {
+                const vocabIdx = this.vocabulary[t];
+                if (document[vocabIdx]) {
+                    document[vocabIdx] += 1;
+                }
+                else {
+                    if (this.documentFrequency[vocabIdx]) {
+                        this.documentFrequency[vocabIdx] += 1;
+                    }
+                    else {
+                        this.documentFrequency[vocabIdx] = 1;
+                    }
+                    document[vocabIdx] = 1;
+                }
+            });
+            this.documents.push(document);
+        });
+    }
+    limit() {
+        const nMaxDF = Math.floor(this.documents.length * this.maxDF);
+        const vocabIdxsToDelete = [];
+        this.documents.forEach(doc => {
+            Object.keys(doc).forEach(vocabIdx => {
+                if (this.documentFrequency[vocabIdx] > nMaxDF) {
+                    delete doc[vocabIdx];
+                    vocabIdxsToDelete.push(vocabIdx);
+                }
+            });
+        });
+        vocabIdxsToDelete.forEach(vocabIdx => {
+            delete this.documentFrequency[vocabIdx];
+            delete this.vocabulary[vocabIdx];
+        });
+    }
+    /**
+     * Smooth idf weights by adding 1 to document frequencies (DF), as if an extra
+     * document was seen containing every term in the collection exactly once.
+     * This prevents zero divisions.
+     * */
+    smooth() {
+        // for each vocabulary
+        Object.values(this.vocabulary).forEach(vocabIdx => (this.documentFrequency[vocabIdx] =
+            this.documentFrequency[vocabIdx] + 1));
+        this.documents.push({ [SMOOTHING_KEY]: 1 });
+    }
+    buildTfidfs() {
+        const tfidfs = [];
+        this.documents.forEach(document => {
+            // this means all the terms in the document are the terms
+            // that have high document frequency.
+            // This will make all the docs with high DF to be clustered together.
+            if (Object.keys(document).length === 0) {
+                tfidfs.push({ [VOCAB_IDX_FOR_DOC_WITH_HIGH_DF]: 1 });
+                return;
+            }
+            if (!document[SMOOTHING_KEY]) {
+                const atfidf = Object.keys(document).map(vocabIdx => {
+                    return [vocabIdx, this.tf(vocabIdx, document) * this.idf(vocabIdx)];
+                });
+                // normalizing the values
+                const dotSum = atfidf
+                    .map(([_, tfidfValue]) => tfidfValue * tfidfValue)
+                    .reduce((sum, tfidfValueSquered) => sum + tfidfValueSquered, 0);
+                const dotSumSqrRoot = Math.sqrt(dotSum);
+                // Normalizing tfidfs
+                const atfidfVocabIdxValueObject = atfidf
+                    .map(([vocabIdx, tfidfValue]) => [
+                    vocabIdx,
+                    tfidfValue / dotSumSqrRoot,
+                ])
+                    .reduce((obj, [vocabIdx, value]) => {
+                    obj[vocabIdx] = value;
+                    return obj;
+                }, {});
+                tfidfs.push(atfidfVocabIdxValueObject);
+            }
+        });
+        return tfidfs;
+    }
+    tf(vocabIdx, document) {
+        return 1 + Math.log(document[vocabIdx]);
+    }
+    idf(vocabIdx) {
+        return (1 + Math.log(this.documents.length / this.documentFrequency[vocabIdx]));
+    }
+}
+exports.TfidfVectorizer = TfidfVectorizer;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memlab/core",
-  "version": "1.1.5",
+  "version": "1.1.9",
   "license": "MIT",
   "description": "memlab core libraries",
   "author": "Liang Gong <lgong@fb.com>",