@oneuptime/common 12.0.31 → 12.0.33

package/Server/Infrastructure/Postgres/SchemaMigrations/1791100000000-AddUserNotificationEmailRollupSetting.ts

@@ -0,0 +1,79 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * The per-user escape hatch from owner email burst rollup: one row per
+ * (user, project) saying whether that person's notification emails may be
+ * coalesced during a burst.
+ *
+ * Two decisions here that a regeneration would silently reverse.
+ *
+ * isEnabled is NOT NULL DEFAULT TRUE, and the table starts empty. Rollup is on
+ * by default and there is no backfill, so almost nobody will ever have a row;
+ * the read path treats an absent row and isEnabled = true identically, which
+ * is what lets this ship without writing a row for a single existing member.
+ * Flip the default to false and every project member silently opts out of a
+ * feature they were never asked about.
+ *
+ * Both foreign keys are ON DELETE CASCADE. A preference about a deleted
+ * project, held by a deleted user, has nothing left to express - and the
+ * tenant column is NOT NULL, so SET NULL could not be satisfied anyway.
+ *
+ * There is deliberately no UNIQUE index on (projectId, userId), even though
+ * the service treats the pair as unique. The dashboard creates the row lazily
+ * on the first toggle and the service's check-then-create narrows the race;
+ * adding the constraint would be a correctness improvement but it would turn
+ * the losing side of that race from a harmless duplicate into a 500 on a
+ * settings page, and both rows say the same thing in every flow the UI can
+ * produce. Worth revisiting if the row ever carries more than one field.
+ */
+export class AddUserNotificationEmailRollupSetting1791100000000
+  implements MigrationInterface
+{
+  public name: string = "AddUserNotificationEmailRollupSetting1791100000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "UserNotificationEmailRollupSetting" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "userId" uuid NOT NULL, "isEnabled" boolean NOT NULL DEFAULT true, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_2d06015d532e73decda61f6c729" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_fc43fd93f6c9b84517ec70a756" ON "UserNotificationEmailRollupSetting" ("projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_0eb2b532417a885a83ef3a246c" ON "UserNotificationEmailRollupSetting" ("userId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" ADD CONSTRAINT "FK_fc43fd93f6c9b84517ec70a756e" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" ADD CONSTRAINT "FK_0eb2b532417a885a83ef3a246cf" FOREIGN KEY ("userId") REFERENCES "User"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" ADD CONSTRAINT "FK_b3e85e262ece458665f3b67ec9c" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" ADD CONSTRAINT "FK_4f8b04650019cb0b9e4cee5f561" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" DROP CONSTRAINT "FK_4f8b04650019cb0b9e4cee5f561"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" DROP CONSTRAINT "FK_b3e85e262ece458665f3b67ec9c"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" DROP CONSTRAINT "FK_0eb2b532417a885a83ef3a246cf"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationEmailRollupSetting" DROP CONSTRAINT "FK_fc43fd93f6c9b84517ec70a756e"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_0eb2b532417a885a83ef3a246c"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_fc43fd93f6c9b84517ec70a756"`,
+    );
+    await queryRunner.query(`DROP TABLE "UserNotificationEmailRollupSetting"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1791200000000-AddNetworkSnmpCredentialProfilesAndAlertPolicies.ts

@@ -0,0 +1,191 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Ping-first network device polling.
+ *
+ *   - NetworkSnmpCredentialProfile: reusable SNMP credentials, resolved for a
+ *     device as device columns -> device profile -> site profile -> ping only.
+ *   - NetworkAlertPolicy: a scope (all / sites / roles / labels) plus a
+ *     Network Device monitor template; the engine keeps one monitor per
+ *     covered device (Monitor.networkAlertPolicyId) so alerting is a policy,
+ *     not something configured device by device.
+ *   - NetworkSite.probeId / snmpCredentialProfileId: the site's monitoring
+ *     defaults, copied onto devices at write time.
+ *   - NetworkDevice.isSnmpReachable / lastSnmpSeenAt: the SNMP walk's own
+ *     result, separate from reachability (which ping now decides).
+ *   - NetworkDevice.monitoringMethod defaults to 'Probe' (formerly 'SNMP');
+ *     the NormalizeNetworkDeviceMonitoringMethod data migration rewrites the
+ *     stored values.
+ */
+export class AddNetworkSnmpCredentialProfilesAndAlertPolicies1791200000000
+  implements MigrationInterface
+{
+  public name: string =
+    "AddNetworkSnmpCredentialProfilesAndAlertPolicies1791200000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "NetworkAlertPolicy" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "description" character varying(500), "isEnabled" boolean NOT NULL DEFAULT true, "monitorTemplateId" uuid, "scope" jsonb NOT NULL DEFAULT '{}', "lastSyncAt" TIMESTAMP WITH TIME ZONE, "lastSyncError" character varying(500), "coveredDeviceCount" integer DEFAULT '0', "templateSyncedAt" TIMESTAMP WITH TIME ZONE, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_a6fa7156f8fafda3d045689f33b" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_784e9f96c6229aadb1f3ff1f4f" ON "NetworkAlertPolicy" ("projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_248ef6126744a645d084cb2431" ON "NetworkAlertPolicy" ("monitorTemplateId") `,
+    );
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "IDX_network_alert_policy_project_template_unique" ON "NetworkAlertPolicy" ("projectId", "monitorTemplateId") WHERE "deletedAt" IS NULL AND "monitorTemplateId" IS NOT NULL`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "NetworkSnmpCredentialProfile" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "description" character varying(500), "snmpVersion" character varying(100) DEFAULT 'V2c', "snmpCommunityString" text, "snmpPort" integer DEFAULT '161', "snmpV3SecurityLevel" character varying(100), "snmpV3Username" character varying(100), "snmpV3AuthProtocol" character varying(100), "snmpV3AuthKey" text, "snmpV3PrivProtocol" character varying(100), "snmpV3PrivKey" text, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_7bac722bd8323e891d3a1c5292f" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_3114361b9ddb6a70d7c538c28d" ON "NetworkSnmpCredentialProfile" ("projectId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Monitor" ADD "networkAlertPolicyId" uuid`,
+    );
+    await queryRunner.query(`ALTER TABLE "NetworkSite" ADD "probeId" uuid`);
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSite" ADD "snmpCredentialProfileId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" ADD "snmpCredentialProfileId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" ADD "isSnmpReachable" boolean`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" ADD "lastSnmpSeenAt" TIMESTAMP WITH TIME ZONE`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" ALTER COLUMN "monitoringMethod" SET DEFAULT 'Probe'`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_51cac50b63aac2d24983010b47" ON "Monitor" ("networkAlertPolicyId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_b54c966057aa58e360a4f2e532" ON "NetworkSite" ("probeId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_79c8d9b3f4f3b3c93bddcbd713" ON "NetworkSite" ("snmpCredentialProfileId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_293e5f4308dec96413427b6739" ON "NetworkDevice" ("projectId", "snmpCredentialProfileId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" ADD CONSTRAINT "FK_784e9f96c6229aadb1f3ff1f4f6" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" ADD CONSTRAINT "FK_248ef6126744a645d084cb24314" FOREIGN KEY ("monitorTemplateId") REFERENCES "MonitorTemplate"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" ADD CONSTRAINT "FK_10debf45ebcfe435a30185a6c0c" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" ADD CONSTRAINT "FK_e7cc6526ed2f1f23a2f24560c65" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Monitor" ADD CONSTRAINT "FK_51cac50b63aac2d24983010b475" FOREIGN KEY ("networkAlertPolicyId") REFERENCES "NetworkAlertPolicy"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSnmpCredentialProfile" ADD CONSTRAINT "FK_3114361b9ddb6a70d7c538c28da" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSnmpCredentialProfile" ADD CONSTRAINT "FK_0c011dba595b5a41a90a899728c" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSnmpCredentialProfile" ADD CONSTRAINT "FK_66f88d052fc53906511384c1f65" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSite" ADD CONSTRAINT "FK_b54c966057aa58e360a4f2e532a" FOREIGN KEY ("probeId") REFERENCES "Probe"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSite" ADD CONSTRAINT "FK_79c8d9b3f4f3b3c93bddcbd7135" FOREIGN KEY ("snmpCredentialProfileId") REFERENCES "NetworkSnmpCredentialProfile"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" ADD CONSTRAINT "FK_7a42f6f231d1f0db76ab884b22d" FOREIGN KEY ("snmpCredentialProfileId") REFERENCES "NetworkSnmpCredentialProfile"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" DROP CONSTRAINT "FK_7a42f6f231d1f0db76ab884b22d"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSite" DROP CONSTRAINT "FK_79c8d9b3f4f3b3c93bddcbd7135"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSite" DROP CONSTRAINT "FK_b54c966057aa58e360a4f2e532a"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSnmpCredentialProfile" DROP CONSTRAINT "FK_66f88d052fc53906511384c1f65"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSnmpCredentialProfile" DROP CONSTRAINT "FK_0c011dba595b5a41a90a899728c"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSnmpCredentialProfile" DROP CONSTRAINT "FK_3114361b9ddb6a70d7c538c28da"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Monitor" DROP CONSTRAINT "FK_51cac50b63aac2d24983010b475"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" DROP CONSTRAINT "FK_e7cc6526ed2f1f23a2f24560c65"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" DROP CONSTRAINT "FK_10debf45ebcfe435a30185a6c0c"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" DROP CONSTRAINT "FK_248ef6126744a645d084cb24314"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkAlertPolicy" DROP CONSTRAINT "FK_784e9f96c6229aadb1f3ff1f4f6"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_293e5f4308dec96413427b6739"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_79c8d9b3f4f3b3c93bddcbd713"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_b54c966057aa58e360a4f2e532"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_51cac50b63aac2d24983010b47"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" ALTER COLUMN "monitoringMethod" SET DEFAULT 'SNMP'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" DROP COLUMN "lastSnmpSeenAt"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" DROP COLUMN "isSnmpReachable"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkDevice" DROP COLUMN "snmpCredentialProfileId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "NetworkSite" DROP COLUMN "snmpCredentialProfileId"`,
+    );
+    await queryRunner.query(`ALTER TABLE "NetworkSite" DROP COLUMN "probeId"`);
+    await queryRunner.query(
+      `ALTER TABLE "Monitor" DROP COLUMN "networkAlertPolicyId"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_3114361b9ddb6a70d7c538c28d"`,
+    );
+    await queryRunner.query(`DROP TABLE "NetworkSnmpCredentialProfile"`);
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_network_alert_policy_project_template_unique"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_248ef6126744a645d084cb2431"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_784e9f96c6229aadb1f3ff1f4f"`,
+    );
+    await queryRunner.query(`DROP TABLE "NetworkAlertPolicy"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -566,6 +566,13 @@ import { AddCertificateReissueRequestedAtToDomains1790300000000 } from "./179030
 import { AddOnCallCalendarFeeds1790400000000 } from "./1790400000000-AddOnCallCalendarFeeds";
 import { AddNetworkDeviceOidTemplate1790500000000 } from "./1790500000000-AddNetworkDeviceOidTemplate";
 import { AddAutoImportRuleOidTemplate1790600000000 } from "./1790600000000-AddAutoImportRuleOidTemplate";
+import { AddNetworkSiteTypeParentHierarchy1790700000000 } from "./1790700000000-AddNetworkSiteTypeParentHierarchy";
+import { AddNetworkDeviceRoleTable1790800000000 } from "./1790800000000-AddNetworkDeviceRoleTable";
+import { AddTelemetryExceptionErrorClass1790900000000 } from "./1790900000000-AddTelemetryExceptionErrorClass";
+import { AddEnterpriseLicenseUsageProvenance1790900000000 } from "./1790900000000-AddEnterpriseLicenseUsageProvenance";
+import { AddUserNotificationEmailRollup1791000000000 } from "./1791000000000-AddUserNotificationEmailRollup";
+import { AddUserNotificationEmailRollupSetting1791100000000 } from "./1791100000000-AddUserNotificationEmailRollupSetting";
+import { AddNetworkSnmpCredentialProfilesAndAlertPolicies1791200000000 } from "./1791200000000-AddNetworkSnmpCredentialProfilesAndAlertPolicies";
 
 export default [
   InitialMigration,
@@ -1136,4 +1143,11 @@ export default [
   AddOnCallCalendarFeeds1790400000000,
   AddNetworkDeviceOidTemplate1790500000000,
   AddAutoImportRuleOidTemplate1790600000000,
+  AddNetworkSiteTypeParentHierarchy1790700000000,
+  AddNetworkDeviceRoleTable1790800000000,
+  AddTelemetryExceptionErrorClass1790900000000,
+  AddEnterpriseLicenseUsageProvenance1790900000000,
+  AddUserNotificationEmailRollup1791000000000,
+  AddUserNotificationEmailRollupSetting1791100000000,
+  AddNetworkSnmpCredentialProfilesAndAlertPolicies1791200000000,
 ];

package/Server/Infrastructure/QueueWorker.ts

@@ -9,11 +9,13 @@ import { Worker } from "bullmq";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import AppMetrics from "../Utils/Telemetry/AppMetrics";
 import TelemetryContext from "../Utils/Telemetry/TelemetryContext";
-import Telemetry, {
-  Span,
-  SpanException,
-  SpanStatusCode,
-} from "../Utils/Telemetry";
+import {
+  COMPONENT_ATTRIBUTE_KEY,
+  TelemetryComponent,
+  UNIT_OF_WORK_ATTRIBUTE_KEY,
+  UnitOfWork,
+} from "../../Types/Telemetry/UnitOfWork";
+import Telemetry, { Span, SpanStatusCode } from "../Utils/Telemetry";
 import Redis from "./Redis";
 import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
 import logger from "../Utils/Logger";
@@ -62,6 +64,14 @@ export default class QueueWorker {
           {
             queueName: queueName,
             jobName: job.name || "unknown",
+            /*
+             * Set EXPLICITLY, never inherited. A job enqueued from inside an
+             * HTTP request would otherwise inherit that request's
+             * "http-request" marker, and ErrorClassResolver would then trust a
+             * user-error classification for work that has no client behind it.
+             */
+            [UNIT_OF_WORK_ATTRIBUTE_KEY]: UnitOfWork.WorkerJob,
+            [COMPONENT_ATTRIBUTE_KEY]: TelemetryComponent.Worker,
             ...TelemetryContext.pickKnownAttributes(job.data),
           },
           () => {
@@ -77,11 +87,31 @@ export default class QueueWorker {
                   await onJobInQueue(job);
                   span.setStatus({ code: SpanStatusCode.OK });
                 } catch (err) {
-                  span.recordException(err as SpanException);
-                  span.setStatus({ code: SpanStatusCode.ERROR });
+                  /*
+                   * Route through the normalizer rather than calling
+                   * span.recordException directly. Two reasons, both real:
+                   *
+                   * 1. The SDK reads `exception.code` before `exception.name`,
+                   *    and every OneUptime ExceptionCode IS an HTTP status, so
+                   *    the raw call typed every failed job's event "400" /
+                   *    "422" — collapsing unrelated failures into a handful of
+                   *    meaningless Issue groups.
+                   * 2. This is the one exception-producing site the ingest
+                   *    drop filter cannot reach, because the raw call set no
+                   *    span attributes for a filter to match on.
+                   *
+                   * Because worker jobs run under unit_of_work="worker-job", a
+                   * user-error class escaping a job is promoted back to
+                   * code-fault — worker BUGS get louder while tenant-config
+                   * noise gets quieter, which is the correct direction.
+                   */
+                  Telemetry.recordExceptionOnSpan({
+                    span,
+                    exception: err,
+                  });
                   throw err;
                 } finally {
-                  span.end();
+                  Telemetry.endSpan(span);
                 }
               },
             });

package/Server/Middleware/BearerTokenAuthorization.ts

@@ -8,6 +8,7 @@ import JSONWebToken from "../Utils/JsonWebToken";
 import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
 import { JSONObject } from "../../Types/JSON";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import SpanUtil from "../Utils/Telemetry/SpanUtil";
 
 export default class BearerTokenAuthorization {
   @CaptureSpan()
@@ -36,6 +37,17 @@ export default class BearerTokenAuthorization {
         "Invalid bearer token, or bearer token not provided.",
       );
     } catch (err) {
+      /*
+       * Record on THIS middleware's own @CaptureSpan span before handing the
+       * error to Express. The decorator sees a normal return (we call
+       * next(err) rather than rethrowing — Express 4 does not catch a
+       * rejection from an async middleware), so its recorder never runs and
+       * without this the error is invisible on the span it actually belongs
+       * to. Goes through SpanUtil so the event is typed by class name rather
+       * than by HTTP status, and so a rejected credential produces a `fault`
+       * event instead of an Issue.
+       */
+      SpanUtil.recordExceptionOnCurrentSpan(err);
       next(err);
     }
   }

package/Server/Middleware/ProjectAuthorization.ts

@@ -281,6 +281,17 @@ export default class ProjectMiddleware {
 
       throw new BadDataException("Invalid Project ID or API Key");
     } catch (err) {
+      /*
+       * Record on THIS middleware's own @CaptureSpan span before handing the
+       * error to Express. The decorator sees a normal return (we call
+       * next(err) rather than rethrowing — Express 4 does not catch a
+       * rejection from an async middleware), so its recorder never runs and
+       * without this the error is invisible on the span it actually belongs
+       * to. Goes through SpanUtil so the event is typed by class name rather
+       * than by HTTP status, and so a rejected credential produces a `fault`
+       * event instead of an Issue.
+       */
+      SpanUtil.recordExceptionOnCurrentSpan(err);
       next(err);
     }
   }

package/Server/Middleware/SCIMAuthorization.ts

@@ -12,6 +12,7 @@ import StatusPageSCIM from "../../Models/DatabaseModels/StatusPageSCIM";
 import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
 import BadRequestException from "../../Types/Exception/BadRequestException";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import SpanUtil from "../Utils/Telemetry/SpanUtil";
 import logger, { getLogAttributesFromRequest } from "../Utils/Logger";
 
 export default class SCIMMiddleware {
@@ -123,6 +124,17 @@ export default class SCIMMiddleware {
         "Invalid bearer token or SCIM configuration not found",
       );
     } catch (err) {
+      /*
+       * Record on THIS middleware's own @CaptureSpan span before handing the
+       * error to Express. The decorator sees a normal return (we call
+       * next(err) rather than rethrowing — Express 4 does not catch a
+       * rejection from an async middleware), so its recorder never runs and
+       * without this the error is invisible on the span it actually belongs
+       * to. Goes through SpanUtil so the event is typed by class name rather
+       * than by HTTP status, and so a rejected credential produces a `fault`
+       * event instead of an Issue.
+       */
+      SpanUtil.recordExceptionOnCurrentSpan(err);
       return next(err);
     }
   }

package/Server/Middleware/TelemetryIngest.ts

@@ -107,6 +107,17 @@ export default class TelemetryIngest {
 
       next();
     } catch (err) {
+      /*
+       * Record on THIS middleware's own @CaptureSpan span before handing the
+       * error to Express. The decorator sees a normal return (we call
+       * next(err) rather than rethrowing — Express 4 does not catch a
+       * rejection from an async middleware), so its recorder never runs and
+       * without this the error is invisible on the span it actually belongs
+       * to. Goes through SpanUtil so the event is typed by class name rather
+       * than by HTTP status, and so a rejected credential produces a `fault`
+       * event instead of an Issue.
+       */
+      SpanUtil.recordExceptionOnCurrentSpan(err);
       return next(err);
     }
   }

package/Server/Services/AIBillingService.ts

@@ -12,6 +12,10 @@ import ObjectID from "../../Types/ObjectID";
 import Project from "../../Models/DatabaseModels/Project";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import SlackUtil from "../Utils/Workspace/Slack/Slack";
+import {
+  BillingFailureNoticeKind,
+  shouldSendBillingFailureNotice,
+} from "../Utils/Billing/BillingFailureNoticeThrottle";
 import URL from "../../Types/API/URL";
 import Exception from "../../Types/Exception/Exception";
 
@@ -24,11 +28,24 @@ export class AIBillingService extends BaseService {
   public async rechargeBalance(
     projectId: ObjectID,
     amountInUSD: number,
+    options?: {
+      /*
+       * See NotificationService.rechargeBalance for the full reasoning. In
+       * short: true for the manual "Recharge" button, where this email is the
+       * only confirmation a person gets that their card was charged, and false
+       * for the automatic top-up, which fires from the balance check that runs
+       * on every AI call.
+       */
+      sendOwnerConfirmationEmail?: boolean | undefined;
+    },
   ): Promise<number> {
     if (!IsBillingEnabled) {
       throw new BadDataException("Billing is not enabled");
     }
 
+    const sendOwnerConfirmationEmail: boolean =
+      options?.sendOwnerConfirmationEmail !== false;
+
     const project: Project | null = await ProjectService.findOneById({
       id: projectId,
       select: {
@@ -52,13 +69,33 @@ export class AIBillingService extends BaseService {
       return 0;
     }
 
+    /*
+     * The no-payment-method branch below throws, and that throw lands in this
+     * method's own catch. Without this the FIRST no-payment-method failure
+     * sends two "ACTION REQUIRED" emails about the same fact, one from each
+     * place.
+     */
+    let ownersAlreadyToldAboutThisFailure: boolean = false;
+
     try {
       if (
         !(await BillingService.hasPaymentMethods(
           project.paymentProviderCustomerId!,
         ))
       ) {
-        if (!project.failedAiBalanceChargeNotificationSentToOwners) {
+        /*
+         * The same 24h window as the catch below. The project boolean this
+         * branch used to latch on is cleared only by a SUCCESSFUL recharge, so
+         * a project that never manages one was told once and then never again.
+         */
+        ownersAlreadyToldAboutThisFailure = true;
+
+        const shouldTellOwners: boolean = await shouldSendBillingFailureNotice({
+          projectId: project.id!,
+          kind: BillingFailureNoticeKind.AiCreditRechargeFailed,
+        });
+
+        if (shouldTellOwners) {
           await ProjectService.updateOneById({
             data: {
               failedAiBalanceChargeNotificationSentToOwners: true,
@@ -114,15 +151,26 @@ export class AIBillingService extends BaseService {
         },
       });
 
-      await ProjectService.sendEmailToProjectOwners(
-        project.id!,
-        "AI Balance Recharge Successful for project - " + (project.name || ""),
-        `We have successfully recharged your AI balance for project - ${
-          project.name || ""
-        } by ${amountInUSD} USD. Your current balance is ${
-          updatedAmount / 100
-        } USD.`,
-      );
+      /*
+       * The confirmation is suppressed for AUTO-recharge only. It used to fire
+       * unconditionally, and AI auto-recharge runs from the balance check on
+       * every AI call, so a busy project mailed every owner once per recharge.
+       * Somebody who deliberately clicked "Recharge" still gets told, because
+       * Project.sendInvoicesByEmail defaults to false and nothing else
+       * reliably confirms the charge.
+       */
+      if (sendOwnerConfirmationEmail) {
+        await ProjectService.sendEmailToProjectOwners(
+          project.id!,
+          "AI Balance Recharge Successful for project - " +
+            (project.name || ""),
+          `We have successfully recharged your AI balance for project - ${
+            project.name || ""
+          } by ${amountInUSD} USD. Your current balance is ${
+            updatedAmount / 100
+          } USD.`,
+        );
+      }
 
       // Send Slack notification for balance refill
       this.sendBalanceRefillSlackNotification({
@@ -139,23 +187,42 @@ export class AIBillingService extends BaseService {
 
       return updatedAmount;
     } catch (err) {
-      await ProjectService.updateOneById({
-        data: {
-          failedAiBalanceChargeNotificationSentToOwners: true,
-        },
-        id: project.id!,
-        props: {
-          isRoot: true,
-        },
-      });
-      await ProjectService.sendEmailToProjectOwners(
-        project.id!,
-        "ACTION REQUIRED: AI Balance Recharge Failed for project - " +
-          (project.name || ""),
-        `We have tried to recharge your AI balance for project - ${
-          project.name || ""
-        } and failed. Please make sure your payment method is up to date and has sufficient balance. You can add new payment methods in Project Settings.`,
-      );
+      /*
+       * This block used to write failedAiBalanceChargeNotificationSentToOwners
+       * and then send WITHOUT ever reading it, so a project with a declined
+       * card mailed every owner once per recharge attempt.
+       *
+       * A 24h window rather than that boolean, for the same reason as the
+       * SMS/call path: the boolean is only cleared by a SUCCESSFUL recharge,
+       * so latching on it would let "no card -> add a card -> the card is
+       * declined" end in permanent silence. See BillingFailureNoticeThrottle.
+       */
+      if (!ownersAlreadyToldAboutThisFailure) {
+        const shouldTellOwners: boolean = await shouldSendBillingFailureNotice({
+          projectId: project.id!,
+          kind: BillingFailureNoticeKind.AiCreditRechargeFailed,
+        });
+
+        if (shouldTellOwners) {
+          await ProjectService.updateOneById({
+            data: {
+              failedAiBalanceChargeNotificationSentToOwners: true,
+            },
+            id: project.id!,
+            props: {
+              isRoot: true,
+            },
+          });
+          await ProjectService.sendEmailToProjectOwners(
+            project.id!,
+            "ACTION REQUIRED: AI Balance Recharge Failed for project - " +
+              (project.name || ""),
+            `We have tried to recharge your AI balance for project - ${
+              project.name || ""
+            } and failed. Please make sure your payment method is up to date and has sufficient balance. You can add new payment methods in Project Settings.`,
+          );
+        }
+      }
       logger.error(err);
       throw err;
     }
@@ -216,6 +283,13 @@ export class AIBillingService extends BaseService {
           const updatedAmount: number = await this.rechargeBalance(
             projectId,
             autoAiRechargeByBalanceInUSD,
+            {
+              /*
+               * This runs from the balance check on every AI call, so a
+               * success email here is one message to every owner per recharge.
+               */
+              sendOwnerConfirmationEmail: false,
+            },
           );
           project.aiCurrentBalanceInUSDCents = updatedAmount;
         }