@oneuptime/common 12.0.14 → 12.0.16

package/build/dist/Utils/Telemetry/LogErrorPattern.js

@@ -0,0 +1,284 @@
+/*
+ * Error-message clustering for the Logs Insights surface.
+ *
+ * "30 x connection refused" is only a useful headline if the thirty raw
+ * bodies — each carrying its own timestamp, request id, port and latency —
+ * collapse onto one row. That collapse is a normalization: every span of
+ * text that varies per occurrence is rewritten to a placeholder, and what
+ * remains is the pattern the occurrences share.
+ *
+ * The rule table below is the single source of truth for that rewrite. It
+ * is consumed twice:
+ *
+ *   - by `normalizeLogBodyToErrorPattern` here, in plain JavaScript, for
+ *     client-side previews and for the test suite; and
+ *   - by Common/Server/Utils/Telemetry/LogErrorPatternSql, which compiles
+ *     the same rules into a nest of ClickHouse `replaceRegexpAll` calls so
+ *     the GROUP BY happens in the database over the whole window rather
+ *     than over a sampled page of rows.
+ *
+ * Because both readers share this table they cannot drift apart, and every
+ * pattern here is therefore restricted to the syntax BOTH engines accept:
+ * RE2 (ClickHouse) has no lookaround and no backreferences, so rules use
+ * only character classes, quantifiers, alternation and `\b`.
+ *
+ * Rule ORDER is part of the contract — rules apply in sequence, each over
+ * the output of the last:
+ *
+ *   1. timestamps first, because their digits and dashes would otherwise be
+ *      shredded by the numeric rule into an unrecognizable shape;
+ *   2. URLs before emails and IPs, so `https://user@10.0.0.1/x` collapses
+ *      to one `<url>` instead of three overlapping placeholders;
+ *   3. long hex runs before numbers, so a trace id stays one token;
+ *   4. numbers last of the value rules, as the catch-all;
+ *   5. whitespace collapse at the end, which is also what flattens a
+ *      multi-line stack trace into a single groupable line.
+ */
+/**
+ * The normalization rules, in application order. See the file header for
+ * why the order matters.
+ *
+ * Replacements are deliberately plain literals: `String.replace` treats
+ * `$&`/`$1` in a replacement specially and ClickHouse treats `\1`
+ * specially, so keeping them free of both characters means the two engines
+ * substitute identical text.
+ */
+export const LOG_ERROR_PATTERN_RULES = [
+    {
+        name: "timestamp",
+        pattern: "\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+-]\\d{2}:?\\d{2})?",
+        replacement: "<timestamp>",
+    },
+    {
+        name: "uuid",
+        pattern: "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",
+        replacement: "<uuid>",
+    },
+    {
+        /*
+         * Before `email` and `ip`: a URL can contain both, and the whole URL is
+         * the varying part.
+         */
+        name: "url",
+        pattern: "[a-zA-Z][a-zA-Z0-9+.-]*://[^\\s\"'<>]+",
+        replacement: "<url>",
+    },
+    {
+        name: "email",
+        pattern: "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}",
+        replacement: "<email>",
+    },
+    {
+        name: "ip",
+        pattern: "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}",
+        replacement: "<ip>",
+    },
+    {
+        /*
+         * Trace ids, span ids, content hashes, object ids. 16 is the shortest
+         * run this fires on so ordinary hex-looking words ("deadbeef", "cafe")
+         * survive as themselves.
+         */
+        name: "hex",
+        pattern: "\\b[0-9a-fA-F]{16,}\\b",
+        replacement: "<hex>",
+    },
+    /*
+     * DELIBERATELY NO "mask any quoted span" RULE HERE.
+     *
+     * A `"[^"]*"` -> `"<str>"` rule looks reasonable on prose like
+     * `unknown field "customerRef"`, and it is catastrophic on structured
+     * logs. In JSON and logfmt — the two commonest shapes there are — the
+     * human-readable message IS a quoted value, so such a rule masks the only
+     * part that identifies the error:
+     *
+     *   {"level":"error","msg":"connection refused","attempt":3}
+     *   {"level":"error","msg":"permission denied","attempt":3}
+     *
+     * would both collapse to `{"<str>":"<str>","<str>":"<str>","<str>":<num>}`,
+     * summing two unrelated failures into one Top Errors row headed by
+     * whichever sample body argMax happened to pick.
+     *
+     * Nothing is lost by leaving it out: every quoted span that genuinely
+     * VARIES is a uuid, number, ip, url, timestamp, hex id or email, and the
+     * rules above already mask those — more informatively, since `"<uuid>"`
+     * says more than `"<str>"`. All such a rule uniquely added was collapsing
+     * quoted WORDS, which is exactly the destructive case. Two different
+     * quoted field names therefore stay two rows: under-collapsing is the
+     * safe direction, because a split incident is still visible while a
+     * merged one is hidden.
+     */
+    {
+        name: "number",
+        pattern: "\\d+(\\.\\d+)?",
+        replacement: "<num>",
+    },
+    {
+        /*
+         * Last: collapses the newlines and indentation of a stack trace onto
+         * one line so the truncation below keeps the exception type and the
+         * top frames — the part that identifies the error.
+         */
+        name: "whitespace",
+        pattern: "\\s+",
+        replacement: " ",
+    },
+];
+/**
+ * Longest pattern kept, in characters.
+ *
+ * A stack trace normalizes to a very long single line; grouping on the
+ * whole thing would split occurrences whose tails differ (different frame
+ * counts, different truncation by the emitting library) while adding
+ * nothing to what identifies the error. 300 characters comfortably covers
+ * an exception type plus its first frames.
+ */
+export const LOG_ERROR_PATTERN_MAX_LENGTH = 300;
+/**
+ * The placeholders the rules can emit. Exported so consumers (the deep-link
+ * builder below, the UI's pattern renderer) can recognize the variable
+ * spans without re-deriving them from the rule table.
+ */
+export const LOG_ERROR_PATTERN_PLACEHOLDERS = LOG_ERROR_PATTERN_RULES.filter((rule) => {
+    return rule.replacement.startsWith("<") && rule.replacement.endsWith(">");
+}).map((rule) => {
+    return rule.replacement;
+});
+/*
+ * Any placeholder token, for splitting a pattern back into its literal
+ * spans. Matches the shape every rule's replacement uses rather than being
+ * built from the table itself, so a rule added later is understood here
+ * without a second edit.
+ */
+const PLACEHOLDER_TOKEN = /<[a-z]+>/g;
+/**
+ * Collapse one raw log body to its error pattern.
+ *
+ * Non-string input (a log row can arrive with a null/absent body) yields
+ * the empty string rather than throwing, because this runs over rows the
+ * caller did not construct.
+ */
+export function normalizeLogBodyToErrorPattern(body) {
+    if (typeof body !== "string" || body.length === 0) {
+        return "";
+    }
+    let normalized = body;
+    for (const rule of LOG_ERROR_PATTERN_RULES) {
+        /*
+         * A replacer function rather than the literal string: it makes the `$`
+         * substitution rules of String.replace unreachable, so the replacement
+         * text is inserted verbatim exactly as ClickHouse inserts it.
+         */
+        normalized = normalized.replace(new RegExp(rule.pattern, "g"), () => {
+            return rule.replacement;
+        });
+    }
+    normalized = normalized.trim();
+    if (normalized.length > LOG_ERROR_PATTERN_MAX_LENGTH) {
+        normalized = normalized.slice(0, LOG_ERROR_PATTERN_MAX_LENGTH).trim();
+    }
+    return normalized;
+}
+/*
+ * Leading punctuation a placeholder leaves behind — `to <ip>:<num>` splits
+ * into `to ` and `:` — so the search text comes out as words, not as the
+ * glue between them.
+ */
+const LEADING_GLUE = /^[\s"':,;=([{]+/;
+/*
+ * Named rather than inlined at the use sites: an inline literal used as
+ * `/\s/.test(x)` trips eslint's wrap-regex, whose fix prettier then undoes
+ * (the same fight CrossSignalScope documents).
+ */
+const ANY_WHITESPACE = /\s/;
+const WHITESPACE_RUN = /\s+/;
+/**
+ * A literal run from the pattern that can be handed to a substring search.
+ *
+ * This is what turns "Insights says this happened 30 times" into a Logs
+ * viewer deep link that actually lists those 30 rows: the viewer filters
+ * `body` with a contains-match, which a pattern containing `<num>` would
+ * never satisfy.
+ *
+ * The subtlety is that a pattern is NOT a substring of the body it came
+ * from, even between placeholders. The last normalization rule collapses
+ * `\s+` to a single space, so any run spanning a line break in the original
+ * — every stack trace — reads `is null at com.example.Service.process(` in
+ * the pattern where the body has `is null\n\tat com.example...`. Handing
+ * that to `body ILIKE '%...%'` matches nothing, landing the user on an
+ * empty list: the precise failure this function exists to avoid.
+ *
+ * So the candidates are ranked longest-first and the first SAFE one wins:
+ *
+ *  - a whitespace-free token is always safe, because there was no
+ *    whitespace inside it to collapse;
+ *  - a longer multi-word run is used only when `sampleBody` proves it
+ *    survived the collapse verbatim.
+ *
+ * Returns "" when nothing usable survives (an all-placeholder pattern, or
+ * runs too short to be selective), which callers read as "no body filter"
+ * rather than as a filter matching everything.
+ */
+export function getErrorPatternSearchText(pattern, options = {}) {
+    var _a;
+    /*
+     * Number form kept for the original `(pattern, minimumLength)` callers.
+     */
+    const resolved = typeof options === "number" ? { minimumLength: options } : options;
+    const minimumLength = (_a = resolved.minimumLength) !== null && _a !== void 0 ? _a : 4;
+    const sampleBody = typeof resolved.sampleBody === "string" ? resolved.sampleBody : "";
+    if (typeof pattern !== "string" || pattern.length === 0) {
+        return "";
+    }
+    const candidates = [];
+    for (const segment of pattern.split(PLACEHOLDER_TOKEN)) {
+        const run = segment.replace(LEADING_GLUE, "").trim();
+        if (run.length > 0) {
+            // The multi-word run: strongest, but only safe when verified.
+            candidates.push(run);
+        }
+        /*
+         * ...and every whitespace-free token inside it, which needs no
+         * verification at all.
+         */
+        for (const token of segment.split(WHITESPACE_RUN)) {
+            const trimmed = token.replace(LEADING_GLUE, "").trim();
+            if (trimmed.length > 0) {
+                candidates.push(trimmed);
+            }
+        }
+    }
+    candidates.sort((a, b) => {
+        return b.length - a.length;
+    });
+    for (const candidate of candidates) {
+        if (candidate.length < minimumLength) {
+            // Sorted longest-first, so everything after this is too short too.
+            break;
+        }
+        const isSingleToken = !ANY_WHITESPACE.test(candidate);
+        if (isSingleToken) {
+            return candidate;
+        }
+        if (sampleBody.length > 0 && sampleBody.includes(candidate)) {
+            return candidate;
+        }
+    }
+    return "";
+}
+/**
+ * A short, single-line label for a pattern, for list rows and chips.
+ * Truncation is by characters with an ellipsis; the full pattern stays
+ * available for tooltips and for the detail panel.
+ */
+export function truncateErrorPattern(pattern, maxLength) {
+    if (typeof pattern !== "string") {
+        return "";
+    }
+    const collapsed = pattern.replace(/\s+/g, " ").trim();
+    if (maxLength <= 0 || collapsed.length <= maxLength) {
+        return collapsed;
+    }
+    return `${collapsed.slice(0, maxLength).trim()}...`;
+}
+//# sourceMappingURL=LogErrorPattern.js.map

package/build/dist/Utils/Telemetry/LogErrorPattern.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LogErrorPattern.js","sourceRoot":"","sources":["../../../../Utils/Telemetry/LogErrorPattern.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAcH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAA+B;IACjE;QACE,IAAI,EAAE,WAAW;QACjB,OAAO,EACL,+EAA+E;QACjF,WAAW,EAAE,aAAa;KAC3B;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EACL,6EAA6E;QAC/E,WAAW,EAAE,QAAQ;KACtB;IACD;QACE;;;WAGG;QACH,IAAI,EAAE,KAAK;QACX,OAAO,EAAE,wCAAwC;QACjD,WAAW,EAAE,OAAO;KACrB;IACD;QACE,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,iDAAiD;QAC1D,WAAW,EAAE,SAAS;KACvB;IACD;QACE,IAAI,EAAE,IAAI;QACV,OAAO,EAAE,2CAA2C;QACpD,WAAW,EAAE,MAAM;KACpB;IACD;QACE;;;;WAIG;QACH,IAAI,EAAE,KAAK;QACX,OAAO,EAAE,wBAAwB;QACjC,WAAW,EAAE,OAAO;KACrB;IACD;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,gBAAgB;QACzB,WAAW,EAAE,OAAO;KACrB;IACD;QACE;;;;WAIG;QACH,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,MAAM;QACf,WAAW,EAAE,GAAG;KACjB;CACF,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAW,GAAG,CAAC;AAExD;;;;GAIG;AACH,MAAM,CAAC,MAAM,8BAA8B,GACzC,uBAAuB,CAAC,MAAM,CAAC,CAAC,IAAyB,EAAW,EAAE;IACpE,OAAO,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAC5E,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAyB,EAAU,EAAE;IAC3C,OAAO,IAAI,CAAC,WAAW,CAAC;AAC1B,CAAC,CAAC,CAAC;AAEL;;;;;GAKG;AACH,MAAM,iBAAiB,GAAW,WAAW,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,UAAU,8BAA8B,CAC5C,IAA+B;IAE/B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,UAAU,GAAW,IAAI,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,uBAAuB,EAAE,CAAC;QAC3C;;;;WAIG;QACH,UAAU,GAAG,UAAU,CAAC,OAAO,CAC7B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,EAC7B,GAAW,EAAE;YACX,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC,CACF,CAAC;IACJ,CAAC;IAED,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAE/B,IAAI,UAAU,CAAC,MAAM,GAAG,4BAA4B,EAAE,CAAC;QACrD,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,4BAA4B,CAAC,CAAC,IAAI,EAAE,CAAC;IACxE,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAYD;;;;GAIG;AACH,MAAM,YAAY,GAAW,iBAAiB,CAAC;AAE/C;;;;GAIG;AACH,MAAM,cAAc,GAAW,IAAI,CAAC;AACpC,MAAM,cAAc,GAAW,KAAK,CAAC;AAErC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,yBAAyB,CACvC,OAAkC,EAClC,UAAkD,EAAE;;IAEpD;;OAEG;IACH,MAAM,QAAQ,GACZ,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IAErE,MAAM,aAAa,GAAW,MAAA,QAAQ,CAAC,aAAa,mCAAI,CAAC,CAAC;IAC1D,MAAM,UAAU,GACd,OAAO,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAErE,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAkB,EAAE,CAAC;IAErC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACvD,MAAM,GAAG,GAAW,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAE7D,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,8DAA8D;YAC9D,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAED;;;WAGG;QACH,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAClD,MAAM,OAAO,GAAW,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAE/D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;QAC/C,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;YACrC,mEAAmE;YACnE,MAAM;QACR,CAAC;QAED,MAAM,aAAa,GAAY,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE/D,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5D,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAkC,EAClC,SAAiB;IAEjB,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAW,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9D,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QACpD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC;AACtD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oneuptime/common",
-  "version": "12.0.14",
+  "version": "12.0.16",
   "description": "",
   "repository": {
     "type": "git",
@@ -32,6 +32,7 @@
     "@types/ejs": "^3.1.1",
     "@types/express": "^4.17.13",
     "@types/jest": "^28.1.4",
+    "@types/js-yaml": "^4.0.9",
     "@types/json2csv": "^5.0.3",
     "@types/jsonwebtoken": "^8.5.9",
     "@types/mssql": "^12.3.0",
@@ -110,6 +111,7 @@
     "i18next": "^23.16.8",
     "ioredis": "^5.3.2",
     "isolated-vm": "^7.0.0",
+    "js-yaml": "^4.1.0",
     "json2csv": "^5.0.7",
     "json5": "^2.2.3",
     "jsonwebtoken": "^9.0.0",