@oneuptime/common 12.0.4 → 12.0.6

package/Server/Infrastructure/ClickhouseDatabase.ts

@@ -87,7 +87,7 @@ export default class ClickhouseDatabase {
             }
 
             logger.debug(
-              `Clickhouse Database Connected: ${dataSourceOptions.host?.toString()}`,
+              `Clickhouse Database Connected: ${dataSourceOptions.url?.toString()}`,
             );
 
             return clickhouseClient;

package/Server/Infrastructure/Postgres/SchemaMigrations/1786096660558-AddTimeRangeToLogSavedView.ts

@@ -0,0 +1,27 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * A saved log view used to carry its time selection only inside the serialized
+ * `query`, where it had already been resolved to an absolute InBetween window.
+ * That froze "Past 1 Hour" into the hour it happened to be when the view was
+ * saved. The selection itself now lives here — the rolling range token, or the
+ * absolute window when the range is Custom — and is re-resolved on every apply.
+ *
+ * Nullable on purpose: rows created before this column still carry their
+ * absolute window in `query`, and readers fall back to it as a Custom range.
+ */
+export class AddTimeRangeToLogSavedView1786096660558
+  implements MigrationInterface
+{
+  public name = "AddTimeRangeToLogSavedView1786096660558";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "LogSavedView" ADD "timeRange" jsonb`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "LogSavedView" DROP COLUMN "timeRange"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786101798351-MigrationName.ts

@@ -0,0 +1,89 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1786101798351 implements MigrationInterface {
+  public name = "MigrationName1786101798351";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableIncidentInstrumentationFixTasks" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAlertInstrumentationFixTasks" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticIncidentCodeFixes" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticAlertCodeFixes" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "incidentAiDailyAutonomousTokenLimit" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "alertAiDailyAutonomousTokenLimit" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "incidentAiDailyFixTaskLimit" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "alertAiDailyFixTaskLimit" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "incidentAiMaxConcurrentInvestigations" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "alertAiMaxConcurrentInvestigations" integer`,
+    );
+    await queryRunner.query(
+      `UPDATE "Project" SET "enableIncidentInstrumentationFixTasks" = "enableInstrumentationFixTasks", "enableAlertInstrumentationFixTasks" = "enableInstrumentationFixTasks", "enableAutomaticIncidentCodeFixes" = "enableAutomaticCodeFixes", "enableAutomaticAlertCodeFixes" = "enableAutomaticCodeFixes", "incidentAiDailyAutonomousTokenLimit" = "aiDailyAutonomousTokenLimit", "alertAiDailyAutonomousTokenLimit" = "aiDailyAutonomousTokenLimit", "incidentAiDailyFixTaskLimit" = "aiDailyFixTaskLimit", "alertAiDailyFixTaskLimit" = "aiDailyFixTaskLimit", "incidentAiMaxConcurrentInvestigations" = "aiMaxConcurrentInvestigations", "alertAiMaxConcurrentInvestigations" = "aiMaxConcurrentInvestigations"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableInstrumentationFixTasks"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticCodeFixes"`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticCodeFixes" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableInstrumentationFixTasks" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `UPDATE "Project" SET "enableInstrumentationFixTasks" = "enableIncidentInstrumentationFixTasks" OR "enableAlertInstrumentationFixTasks", "enableAutomaticCodeFixes" = "enableAutomaticIncidentCodeFixes" OR "enableAutomaticAlertCodeFixes"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "alertAiMaxConcurrentInvestigations"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "incidentAiMaxConcurrentInvestigations"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "alertAiDailyFixTaskLimit"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "incidentAiDailyFixTaskLimit"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "alertAiDailyAutonomousTokenLimit"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "incidentAiDailyAutonomousTokenLimit"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticAlertCodeFixes"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticIncidentCodeFixes"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAlertInstrumentationFixTasks"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableIncidentInstrumentationFixTasks"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786105470826-MigrationName.ts

@@ -0,0 +1,27 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1786105470826 implements MigrationInterface {
+  public name = "MigrationName1786105470826";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "IncidentFeed" ADD "aiRunId" uuid`);
+    await queryRunner.query(`ALTER TABLE "AlertFeed" ADD "aiRunId" uuid`);
+    await queryRunner.query(
+      `CREATE INDEX "IDX_bae064a2cf9282fc2cca934d78" ON "IncidentFeed" ("incidentId", "aiRunId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_7da528b06c0ae8b2f1b6c9c848" ON "AlertFeed" ("alertId", "aiRunId") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_7da528b06c0ae8b2f1b6c9c848"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_bae064a2cf9282fc2cca934d78"`,
+    );
+    await queryRunner.query(`ALTER TABLE "AlertFeed" DROP COLUMN "aiRunId"`);
+    await queryRunner.query(`ALTER TABLE "IncidentFeed" DROP COLUMN "aiRunId"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786303472848-AddDataSourceTable.ts

@@ -0,0 +1,39 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddDataSourceTable1786303472848 implements MigrationInterface {
+  public name: string = "AddDataSourceTable1786303472848";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "DataSource" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "slug" character varying(100) NOT NULL, "description" character varying(500), "dataSourceType" character varying(100) NOT NULL, "url" character varying, "databaseHost" character varying(100), "databasePort" integer, "databaseName" character varying(100), "username" character varying(100), "password" text, "apiToken" text, "customHeaders" jsonb, "additionalOptions" jsonb, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_49141becafea4a0574d9b5219ca" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_0e559ea9d7fbc66d014a29c380" ON "DataSource" ("projectId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "DataSource" ADD CONSTRAINT "FK_0e559ea9d7fbc66d014a29c3804" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "DataSource" ADD CONSTRAINT "FK_38843f53e5ca0ce0bf4de525d6f" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "DataSource" ADD CONSTRAINT "FK_29f0d663494b52c4ed2c0cc8e88" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "DataSource" DROP CONSTRAINT "FK_29f0d663494b52c4ed2c0cc8e88"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "DataSource" DROP CONSTRAINT "FK_38843f53e5ca0ce0bf4de525d6f"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "DataSource" DROP CONSTRAINT "FK_0e559ea9d7fbc66d014a29c3804"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_0e559ea9d7fbc66d014a29c380"`,
+    );
+    await queryRunner.query(`DROP TABLE "DataSource"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786400000000-AddMasterPasswordSalt.ts

@@ -0,0 +1,40 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+type ScryptHashExistsResult = {
+  exists: boolean;
+};
+
+export class AddMasterPasswordSalt1786400000000 implements MigrationInterface {
+  public name = "AddMasterPasswordSalt1786400000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "StatusPage" ADD "masterPasswordSalt" character varying(100)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Dashboard" ADD "masterPasswordSalt" character varying(100)`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    const [dashboardResult] = (await queryRunner.query(
+      `SELECT EXISTS (SELECT 1 FROM "Dashboard" WHERE "masterPassword" LIKE 'scrypt$%') AS "exists"`,
+    )) as Array<ScryptHashExistsResult>;
+    const [statusPageResult] = (await queryRunner.query(
+      `SELECT EXISTS (SELECT 1 FROM "StatusPage" WHERE "masterPassword" LIKE 'scrypt$%') AS "exists"`,
+    )) as Array<ScryptHashExistsResult>;
+
+    if (dashboardResult?.exists || statusPageResult?.exists) {
+      throw new Error(
+        "Cannot remove master-password salt columns while scrypt master-password hashes exist.",
+      );
+    }
+
+    await queryRunner.query(
+      `ALTER TABLE "Dashboard" DROP COLUMN "masterPasswordSalt"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "StatusPage" DROP COLUMN "masterPasswordSalt"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786500000000-AddInvestigationCodeFixRecommendation.ts

@@ -0,0 +1,26 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Generated with `npm run generate-postgres-migration` after adding
+ * AIRun.codeFixRecommendation. Existing rows fail closed as NotRecommended;
+ * incident/alert investigations move to Pending only in their atomic
+ * Completed transition, then settle through structured post-analysis
+ * classification.
+ */
+export class AddInvestigationCodeFixRecommendation1786500000000
+  implements MigrationInterface
+{
+  public name = "AddInvestigationCodeFixRecommendation1786500000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" ADD "codeFixRecommendation" character varying(100) NOT NULL DEFAULT 'NotRecommended'`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" DROP COLUMN "codeFixRecommendation"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786600000000-AddInvestigationAnalysisTldr.ts

@@ -0,0 +1,24 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Generated with `npm run generate-postgres-migration` after adding
+ * AIRun.analysisTldr — the AI-written one or two sentence summary of a
+ * completed investigation's posted report. Nullable: legacy runs, runs that
+ * published no report, and runs whose bounded TL;DR call failed simply have
+ * none, and the panel omits the TL;DR block for them.
+ */
+export class AddInvestigationAnalysisTldr1786600000000
+  implements MigrationInterface
+{
+  public name = "AddInvestigationAnalysisTldr1786600000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIRun" ADD "analysisTldr" character varying(500)`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "AIRun" DROP COLUMN "analysisTldr"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1786700000000-DropProjectCallSMSConfigCredentialUniques.ts

@@ -0,0 +1,76 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Drops the global UNIQUE constraints on ProjectCallSMSConfig's Twilio
+ * credentials. Generated with `npm run generate-postgres-migration` after
+ * flipping both columns to `unique: false`; the statements are the generator's,
+ * renumbered to sort last and given IF EXISTS (see below).
+ *
+ * WHY THEY WERE WRONG
+ *
+ * ProjectCallSMSConfig is tenant-scoped (@TenantColumn("projectId")) and
+ * inherently multi-row: a project has as many configs as it has Twilio
+ * setups. The constraints came from the InitialMigration, where this table's
+ * columns were modelled on GlobalConfig — a singleton table whose every column
+ * carries a pointless `unique: true`. On a singleton that is harmless. Here it
+ * meant no two rows anywhere in the instance could share a Twilio account.
+ *
+ * That is not a constraint anybody wants:
+ *
+ *   - One Twilio account, two numbers. Poland (and several other countries)
+ *     requires a Local number for Voice and a Mobile number for SMS, so a
+ *     single project legitimately needs two configs on one account. This is
+ *     what issue #3020 was reported against.
+ *   - Two projects, one account. On a shared instance the FIRST project to
+ *     save a given account SID locked every other project out of it, across
+ *     the tenant boundary — one tenant could deny another the use of its own
+ *     Twilio credentials, and neither could see why.
+ *
+ * Both surfaced the same way: Postgres raised unique_violation (23505),
+ * which is not a OneUptime Exception, so it fell through to the generic
+ * handler and reached the client as a bare 500 `{"error":"Server Error"}`
+ * with nothing to act on.
+ *
+ * NOT REPLACED WITH A NARROWER UNIQUE
+ *
+ * There is no uniqueness to restore at any scope. Reusing one account across
+ * configs is the supported case, not an accident to be guarded against. Config
+ * identity is `name`, which is already unique per project via
+ * @UniqueColumnBy("projectId"). After this migration ProjectCallSMSConfig has
+ * no UNIQUE constraint on any credential column, which matches every other
+ * tenant-scoped table: theirs are all on slugs, keys and tokens — identifiers
+ * that really are global.
+ *
+ * IF EXISTS
+ *
+ * Self-hosted operators hit by #3020 were told in the issue thread that the
+ * workaround is to drop the constraint by hand. IF EXISTS keeps this migration
+ * from failing on those databases. down() cannot be made safe the same way: it
+ * will fail if duplicates exist by then, which is correct — silently
+ * hard-deleting a project's Twilio config to make room for a constraint would
+ * be worse than a failed rollback.
+ */
+export class DropProjectCallSMSConfigCredentialUniques1786700000000
+  implements MigrationInterface
+{
+  public name: string =
+    "DropProjectCallSMSConfigCredentialUniques1786700000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "ProjectCallSMSConfig" DROP CONSTRAINT IF EXISTS "UQ_0886139eac04ad49627e446d477"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProjectCallSMSConfig" DROP CONSTRAINT IF EXISTS "UQ_2eb1a240d549a7701b6e82d2f94"`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "ProjectCallSMSConfig" ADD CONSTRAINT "UQ_2eb1a240d549a7701b6e82d2f94" UNIQUE ("twilioAuthToken")`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProjectCallSMSConfig" ADD CONSTRAINT "UQ_0886139eac04ad49627e446d477" UNIQUE ("twilioAccountSID")`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -501,9 +501,17 @@ import { AddRemediationVerification1785768089408 } from "./1785768089408-AddReme
 import { AddStatusPageMonitorRule1786005052769 } from "./1786005052769-AddStatusPageMonitorRule";
 import { AddPerUserPasswordSalt1786018109307 } from "./1786018109307-AddPerUserPasswordSalt";
 import { WidenHashedStringColumnsForScrypt1786023262402 } from "./1786023262402-WidenHashedStringColumnsForScrypt";
+import { AddTimeRangeToLogSavedView1786096660558 } from "./1786096660558-AddTimeRangeToLogSavedView";
 import { RestoreServiceLowerNameIndex1786100000000 } from "./1786100000000-RestoreServiceLowerNameIndex";
+import { MigrationName1786101798351 } from "./1786101798351-MigrationName";
+import { MigrationName1786105470826 } from "./1786105470826-MigrationName";
 import { RestoreDroppedUniqueIndexes1786200000000 } from "./1786200000000-RestoreDroppedUniqueIndexes";
 import { QuarantineUnboundGitHubInstallations1786300000000 } from "./1786300000000-QuarantineUnboundGitHubInstallations";
+import { AddMasterPasswordSalt1786400000000 } from "./1786400000000-AddMasterPasswordSalt";
+import { AddInvestigationCodeFixRecommendation1786500000000 } from "./1786500000000-AddInvestigationCodeFixRecommendation";
+import { AddInvestigationAnalysisTldr1786600000000 } from "./1786600000000-AddInvestigationAnalysisTldr";
+import { DropProjectCallSMSConfigCredentialUniques1786700000000 } from "./1786700000000-DropProjectCallSMSConfigCredentialUniques";
+import { AddDataSourceTable1786303472848 } from "./1786303472848-AddDataSourceTable";
 
 export default [
   InitialMigration,
@@ -1009,7 +1017,15 @@ export default [
   AddStatusPageMonitorRule1786005052769,
   AddPerUserPasswordSalt1786018109307,
   WidenHashedStringColumnsForScrypt1786023262402,
+  AddTimeRangeToLogSavedView1786096660558,
   RestoreServiceLowerNameIndex1786100000000,
   RestoreDroppedUniqueIndexes1786200000000,
   QuarantineUnboundGitHubInstallations1786300000000,
+  MigrationName1786101798351,
+  AddMasterPasswordSalt1786400000000,
+  MigrationName1786105470826,
+  AddInvestigationCodeFixRecommendation1786500000000,
+  AddInvestigationAnalysisTldr1786600000000,
+  DropProjectCallSMSConfigCredentialUniques1786700000000,
+  AddDataSourceTable1786303472848,
 ];

package/Server/Infrastructure/Queue.ts

@@ -560,6 +560,32 @@ export default class Queue {
     return waitingCount + activeCount + delayedCount;
   }
 
+  /*
+   * Backlog = jobs still waiting for a worker (waiting + delayed), for
+   * autoscaling signals. ACTIVE jobs are deliberately excluded: they are
+   * already being served, so to a scaler they are capacity, not demand.
+   * Telemetry jobs in particular sit in the active state for the fan-in
+   * writer's full flush window (up to TELEMETRY_FANIN_MAX_WAIT_MS) while
+   * awaiting their ClickHouse ack, so a fleet scaled on an active-inclusive
+   * count converges on pod count = job rate x ack latency — every new pod
+   * just parks more jobs without shortening the flush window. Scaling on
+   * waiting + delayed only breaks that feedback loop. getQueueSize above
+   * keeps the active-inclusive sum for ingest backpressure checks, where
+   * in-flight work genuinely counts against capacity.
+   */
+  @CaptureSpan()
+  public static async getQueueBacklogSize(
+    queueName: QueueName,
+  ): Promise<number> {
+    const queue: BullQueue = this.getQueue(queueName);
+    const [waitingCount, delayedCount]: [number, number] = await Promise.all([
+      queue.getWaitingCount(),
+      queue.getDelayedCount(),
+    ]);
+
+    return waitingCount + delayedCount;
+  }
+
   @CaptureSpan()
   public static async getQueueStats(queueName: QueueName): Promise<{
     waiting: number;

package/Server/Infrastructure/QueueWorker.ts

@@ -16,6 +16,7 @@ import Telemetry, {
 } from "../Utils/Telemetry";
 import Redis from "./Redis";
 import GracefulShutdown, { ShutdownPriority } from "../Utils/GracefulShutdown";
+import logger from "../Utils/Logger";
 
 export default class QueueWorker {
   @CaptureSpan()
@@ -120,6 +121,22 @@ export default class QueueWorker {
         : {}),
     });
 
+    /*
+     * Always log job failures to the container log, independent of telemetry
+     * configuration. Error visibility used to ride on @CaptureSpan's
+     * exception path, but the decorator is a passthrough when no span
+     * exporter is installed — without this listener a job that throws on
+     * such a deployment lands in the Redis failed set with no log line.
+     */
+    worker.on("failed", (job: QueueJob | undefined, error: Error) => {
+      logger.error(
+        `Queue job failed: ${queueName}/${job?.name || "unknown"} (job ${
+          job?.id ?? "unknown"
+        })`,
+      );
+      logger.error(error);
+    });
+
     /*
      * Stop pulling new jobs and let in-flight ones finish on shutdown. Runs in
      * the Workers tier — before datastores are drained — so jobs mid-flight can

package/Server/Middleware/TelemetryIngest.ts

@@ -73,16 +73,21 @@ export default class TelemetryIngest {
         );
 
       if (!projectId) {
+        /*
+         * The token value itself is deliberately NOT logged: ingestion
+         * keys are secrets, and a mistyped-but-nearly-valid token in a
+         * log line is a credential leak. The request attributes give
+         * operators enough to correlate with the sender.
+         */
         logger.error(
-          "Invalid service token: " + oneuptimeToken,
+          "Invalid service token.",
           getLogAttributesFromRequest(req as any),
         );
 
         /*
          * 401 is deliberate (see the missing-token branch above): a
          * silent 200 drops the payload while the client believes the
-         * export succeeded. The token value is logged server-side but
-         * intentionally not echoed back in the response body.
+         * export succeeded.
          */
         return Response.sendErrorResponse(
           req,