@oneuptime/common 12.0.31 → 12.0.33

package/Server/Utils/Telemetry.ts

@@ -52,6 +52,12 @@ import { AppVersion, Env, DisableTelemetry } from "../EnvironmentConfig";
 import logger from "./Logger";
 import GracefulShutdown, { ShutdownPriority } from "./GracefulShutdown";
 import ContextSpanProcessor from "./Telemetry/ContextSpanProcessor";
+import ErrorClassResolver from "./Telemetry/ErrorClassResolver";
+import ErrorClass, {
+  isNonActionableErrorClass,
+  markErrorReported,
+} from "../../Types/Telemetry/ErrorClass";
+import { ERROR_CLASS_ATTRIBUTE_KEY } from "../../Types/Telemetry/UnitOfWork";
 import RuntimeMetrics from "./Telemetry/RuntimeMetrics";
 
 type ResourceWithRawAttributes = Resource & {
@@ -540,70 +546,158 @@ export default class Telemetry {
     return this.getTracer().startActiveSpan(name, data.options || {}, data.fn);
   }
 
+  /*
+   * Span event names. The exception name is load-bearing: the ingest side
+   * mints an ExceptionInstance row and a TelemetryException group for every
+   * span event literally named "exception" (OtelTracesIngestService
+   * getSpanEvents), with no severity, status or handled/unhandled filter. An
+   * event under any OTHER name is still pushed to the span's `events` column
+   * and still renders in the Traces UI — it simply never becomes an Issue.
+   *
+   * That is the entire mechanism by which a user error stops paging us, and
+   * it needs no ingest change, no schema change and no migration.
+   */
+  private static readonly SPAN_EVENT_EXCEPTION: string = "exception";
+  private static readonly SPAN_EVENT_FAULT: string = "fault";
+
+  private static faultCounter: TelemetryCounter | null = null;
+
+  /**
+   * Record a thrown value on a span AND end the span.
+   *
+   * Kept as the name every @CaptureSpan frame calls, so the decorator's
+   * contract is unchanged.
+   */
   public static recordExceptionMarkSpanAsErrorAndEndSpan(data: {
     span: Span;
     exception: unknown;
+  }): void {
+    try {
+      this.recordExceptionOnSpan(data);
+    } finally {
+      this.endSpan(data.span);
+    }
+  }
+
+  /**
+   * Record a thrown value on a span WITHOUT ending it, for callers that own
+   * the span's lifetime (QueueWorker's root job span ends it in its own
+   * `finally`).
+   *
+   * What this does, and why each part matters:
+   *
+   * 1. RESOLVES A FAULT DOMAIN. code-fault and infrastructure are real
+   *    failures and behave exactly as before. user-error and expected-denial
+   *    are the platform correctly refusing a request, so they get a `fault`
+   *    event instead of an `exception` event and the span status is left
+   *    alone.
+   *
+   * 2. REPORTS ONCE PER THROWN VALUE. One error crossing N decorated frames
+   *    used to emit N exception events and N logger.error lines. The
+   *    fingerprint hashes no spanId and no span name, so all N landed in the
+   *    same group and occuranceCount jumped by the depth of the call stack
+   *    (measured: 3 for a create with a missing field, 6 for a
+   *    permission-denied get-list). Outer frames still set ERROR status and
+   *    still carry the queryable exception.* attributes, so the trace shows
+   *    the full error path — only the duplicate EVENT and the duplicate LOG
+   *    are suppressed.
+   *
+   * 3. BUILDS THE EVENT ITSELF rather than calling span.recordException. The
+   *    SDK reads `exception.code` BEFORE `exception.name` when deciding
+   *    `exception.type` (sdk-trace-base Span.recordException), and every
+   *    OneUptime ExceptionCode IS an HTTP status — so exceptions were being
+   *    typed "401"/"400"/"500". Constructing the event here removes that trap
+   *    structurally instead of relying on every caller to normalize first.
+   *
+   * 4. COUNTS EVERYTHING. `oneuptime.fault.count` is incremented for every
+   *    class, suppressed or not, so suppression is never invisible. A metric
+   *    also outlives trace retention, which a span event does not.
+   *
+   * Never throws: this is the universal catch path, and a throw here would
+   * mask the original error.
+   */
+  public static recordExceptionOnSpan(data: {
+    span: Span;
+    exception: unknown;
   }): void {
     const { span, exception } = data;
 
-    /*
-     * This runs on the universal catch path of every @CaptureSpan-decorated
-     * function, so it must NEVER throw — a throw here would mask the original
-     * error and skip marking the span. The whole body is wrapped defensively
-     * and the span is always marked-as-error and ended (in the finally).
-     */
     try {
       const exceptionAttributes: Attributes =
         this.getExceptionAttributes(exception);
 
-      // log the exception as well
-      logger.error(exception);
+      const errorClass: ErrorClass = ErrorClassResolver.resolve(exception);
+      const alreadyReported: boolean = markErrorReported(exception);
 
-      /*
-       * Span *events* (from recordException) are not reliably surfaced when the
-       * span is read back, and setStatus on its own only records the error CODE,
-       * not the message. So we also attach the exception details as queryable
-       * span attributes — including DB driver fields like the failing constraint
-       * and table — so the actual cause is visible in the trace UI instead of an
-       * empty "Error" status.
-       */
-      span.setAttributes(exceptionAttributes);
+      const exceptionType: string =
+        (exceptionAttributes["exception.type"] as string) || "Error";
+      const exceptionMessage: string =
+        (exceptionAttributes["exception.message"] as string) || "";
+
+      if (!alreadyReported) {
+        // Logger applies the same classification and demotes to WARN severity.
+        logger.error(exception);
+      }
 
       /*
-       * A NORMALIZED {name, message, stack} is handed to recordException
-       * rather than the raw thrown value, because the OTel SDK reads
-       * `exception.code` BEFORE `exception.name` when deciding
-       * `exception.type` (sdk-trace-base Span.recordException). OneUptime's
-       * Exception base class exposes an HTTP STATUS as `code`
-       * (NotAuthenticatedException = 401, BadDataException = 400,
-       * ServerException = 500 — see Types/Exception/ExceptionCode), so every
-       * exception this platform raises was being typed "401"/"400"/"500".
-       *
-       * That type is the exception event's identity downstream: it is hashed
-       * into the TelemetryException group fingerprint and it is what the
-       * Issues list and the AI insight titles show. A whole service's worth of
-       * unrelated failures collapsed into "401", telling nobody anything.
-       *
-       * The payload is built from the attributes getExceptionAttributes has
-       * already extracted, which means it inherits their truncation (4k
-       * message, 8k stack) and their crash-safety — a hostile thrown value is
-       * read exactly once, defensively, rather than twice.
+       * Span *events* are not reliably surfaced when the span is read back,
+       * and setStatus on its own only records the error CODE, not the message.
+       * So the exception details are also attached as queryable span
+       * attributes — including DB driver fields like the failing constraint
+       * and table — so the actual cause is visible in the trace UI instead of
+       * an empty "Error" status. Done for every class, on every frame: it is
+       * what makes `attributes.exception.type` usable as a drop-filter
+       * predicate at any depth.
        */
-      span.recordException({
-        name: (exceptionAttributes["exception.type"] as string) || "Error",
-        message: (exceptionAttributes["exception.message"] as string) || "",
-        ...(exceptionAttributes["exception.stacktrace"]
-          ? {
-              stack: exceptionAttributes["exception.stacktrace"] as string,
-            }
-          : {}),
-      } as SpanException);
-      span.setStatus({
-        code: SpanStatusCode.ERROR,
-        message:
-          (exceptionAttributes["exception.message"] as string | undefined) ||
-          "Error",
+      span.setAttributes({
+        ...exceptionAttributes,
+        [ERROR_CLASS_ATTRIBUTE_KEY]: errorClass,
       });
+
+      if (!isNonActionableErrorClass(errorClass)) {
+        if (!alreadyReported) {
+          span.addEvent(this.SPAN_EVENT_EXCEPTION, {
+            "exception.type": exceptionType,
+            "exception.message": exceptionMessage,
+            ...(exceptionAttributes["exception.stacktrace"]
+              ? {
+                  "exception.stacktrace": exceptionAttributes[
+                    "exception.stacktrace"
+                  ] as string,
+                }
+              : {}),
+          });
+        }
+
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: exceptionMessage || "Error",
+        });
+      } else if (!alreadyReported) {
+        /*
+         * Deliberately NOT named "exception" and deliberately carrying no
+         * `exception.*` keys: the first stops the trace ingest path from
+         * building an Issue, the second stops LogExceptionExtractor Path A
+         * from resurrecting one out of a log record. The event is still
+         * stored and still rendered.
+         *
+         * No setStatus at all. OTel says a 4xx on a SERVER span must stay
+         * Unset, and setStatus({code: UNSET}) is a documented no-op — so
+         * "leave it unset" is implemented as "do not call it".
+         */
+        span.addEvent(this.SPAN_EVENT_FAULT, {
+          [ERROR_CLASS_ATTRIBUTE_KEY]: errorClass,
+          "error.type": exceptionType,
+          "error.message": exceptionMessage,
+          ...(exceptionAttributes["exception.code"]
+            ? {
+                "error.code": exceptionAttributes["exception.code"] as string,
+              }
+            : {}),
+        });
+      }
+
+      this.incrementFaultCounter(errorClass, exceptionType);
     } catch {
       // Enrichment failed on some exotic thrown value — still flag the span.
       try {
@@ -611,8 +705,34 @@ export default class Telemetry {
       } catch {
         // span may already be ended; nothing more we can do.
       }
-    } finally {
-      this.endSpan(span);
+    }
+  }
+
+  /*
+   * Built lazily and through Telemetry.getCounter directly rather than through
+   * AppMetrics — AppMetrics imports Telemetry, so registering there would
+   * close an import cycle on the error path.
+   */
+  private static incrementFaultCounter(
+    errorClass: ErrorClass,
+    exceptionType: string,
+  ): void {
+    try {
+      if (!this.faultCounter) {
+        this.faultCounter = this.getCounter({
+          name: "oneuptime.fault.count",
+          description:
+            "Faults by fault domain and type, including the ones deliberately kept out of the Issues list.",
+          unit: "1",
+        });
+      }
+
+      this.faultCounter.add(1, {
+        [ERROR_CLASS_ATTRIBUTE_KEY]: errorClass,
+        "error.type": exceptionType,
+      });
+    } catch {
+      // Metrics must never break the error path.
     }
   }
 
@@ -783,7 +903,34 @@ export default class Telemetry {
     }
   }
 
+  /*
+   * Ending a span twice makes the SDK log
+   * "You can only call end() on a span once" via diag. That happened on EVERY
+   * error path: the recorder's finally ended the span, and CaptureSpan then
+   * ended it again in its own finally / .finally(). Guard here rather than at
+   * the call sites so any future caller is safe too.
+   */
   public static endSpan(span: Span): void {
+    try {
+      const endTime: unknown = (span as unknown as { endTime?: Array<number> })
+        .endTime;
+
+      /*
+       * sdk-trace-base sets endTime to [0, 0] until the span ends. A span
+       * implementation that does not expose it (a no-op span, a test double)
+       * falls through and is ended normally.
+       */
+      if (
+        Array.isArray(endTime) &&
+        endTime.length === 2 &&
+        !(endTime[0] === 0 && endTime[1] === 0)
+      ) {
+        return;
+      }
+    } catch {
+      // Fall through and end it.
+    }
+
     span.end();
   }
 }

package/Server/Views/Partials/SensitiveUrlToken.ejs

@@ -0,0 +1,140 @@
+<%#
+    Takes single-use bearer tokens out of the address bar before any
+    third-party tag can read them.
+
+    Password reset and email verification links carry their token in the URL
+    *path* — /accounts/reset-password/<token>, /accounts/verify-email/<token>,
+    /reset-password/<token> on a status page. Every analytics and advertising
+    tag on the page reports the page URL as a matter of course, so the token
+    left the browser to whoever the container happened to be loading. Measured
+    on production before this partial existed, a single visit to
+    /accounts/reset-password/<token> sent the token to five third parties:
+    region1.analytics.google.com, www.google.com, bat.bing.com, q.quora.com and
+    alb.reddit.com. A reset token is a bearer credential for the account, so
+    anyone with read access to any of those ad or analytics dashboards — inside
+    OneUptime or at the vendor — could take the account over for as long as the
+    token stayed valid.
+
+    Two mechanisms, in this order:
+
+    1.  The token is moved into sessionStorage and `history.replaceState`
+        rewrites the path to the token-free route. This runs synchronously in
+        <head>, before the Google Tag Manager snippet below it, so every tag the
+        container loads sees only /accounts/reset-password. Analytics is NOT
+        disabled by this — the signup funnel still reports — it just no longer
+        sees the credential. It also keeps the token out of the browser's own
+        history, which replaceState overwrites rather than appends to.
+
+    2.  If step 1 did not finish — no History API, a storage exception, a URL
+        the parser rejected — the token is still in the address bar, and the
+        partial fails closed by setting
+        __ONEUPTIME_SENSITIVE_URL_TOKEN_PENDING__. The Google Tag Manager
+        snippet on each page checks that flag and does not start the container
+        at all. Losing a page view is the cheaper failure.
+
+    The reset flow itself keeps working in both cases: the client reads the
+    token through Common/UI/Utils/SensitiveUrlToken, which prefers the
+    stash and falls back to the path when the bootstrap could not clean it.
+
+    MUST be included as the first thing in <head>, ahead of Google Tag Manager
+    and ahead of every external <script>. A tag that loads before this runs is
+    a tag that sees the token. Common/Tests/App/SensitiveUrlTokenBootstrap.test.ts
+    asserts that ordering for each page that includes it.
+
+    TOKEN_ROUTES below is duplicated in Common/UI/Utils/SensitiveUrlToken.ts,
+    which has to agree about which routes carry a token; the same test asserts
+    the two lists match. Add a route to both or to neither.
+-%>
+<meta name="referrer" content="no-referrer" />
+<script>
+    (function (win) {
+        var STORAGE_KEY = "oneuptime-sensitive-url-token";
+
+        /*
+         * Route segments whose NEXT path segment is a single-use bearer token.
+         * Matched against the second-to-last segment, so this covers
+         * /accounts/reset-password/<token>, /reset-password/<token> and
+         * /status-page/<id>/reset-password/<token> alike.
+         */
+        var TOKEN_ROUTES = ["reset-password", "verify-email"];
+
+        function segmentsOf(pathname) {
+            var parts = String(pathname || "").split("/");
+            var kept = [];
+
+            for (var i = 0; i < parts.length; i++) {
+                if (parts[i] !== "") {
+                    kept.push(parts[i]);
+                }
+            }
+
+            return kept;
+        }
+
+        /* True when the last segment of this path is a bearer token. */
+        function carriesToken(pathname) {
+            var segments = segmentsOf(pathname);
+
+            if (segments.length < 2) {
+                return false;
+            }
+
+            var parent = segments[segments.length - 2];
+
+            for (var i = 0; i < TOKEN_ROUTES.length; i++) {
+                if (parent === TOKEN_ROUTES[i]) {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        var pending = true;
+
+        try {
+            var url = new URL(win.location.href);
+
+            if (carriesToken(url.pathname)) {
+                var segments = segmentsOf(url.pathname);
+                var token = segments.pop();
+
+                try {
+                    token = decodeURIComponent(token);
+                } catch (_decodeError) {
+                    /* A malformed escape sequence is handed on verbatim. */
+                }
+
+                try {
+                    win.sessionStorage.setItem(STORAGE_KEY, token);
+                } catch (_storageError) {
+                    /*
+                     * Private mode and blocked storage. The in-memory handoff
+                     * does not survive a reload, but it does survive the SPA
+                     * boot, which is all the flow needs.
+                     */
+                    win.__ONEUPTIME_SENSITIVE_URL_TOKEN__ = token;
+                }
+
+                win.history.replaceState(
+                    win.history.state,
+                    "",
+                    "/" + segments.join("/") + url.search + url.hash
+                );
+            }
+
+            /*
+             * Read the address bar back rather than trusting the rewrite. This
+             * is the whole safety condition: analytics runs only if the token
+             * is demonstrably no longer in the URL those tags are about to
+             * report.
+             */
+            pending = carriesToken(win.location.pathname);
+        } catch (_bootstrapError) {
+            /* Fail closed: something went wrong, so assume the token is exposed. */
+            pending = true;
+        }
+
+        win.__ONEUPTIME_SENSITIVE_URL_TOKEN_PENDING__ = pending;
+    })(window);
+</script>