@oneuptime/common 14.0.11 → 14.0.12

package/Server/Services/IncomingCallMissedCallNotificationService.ts

@@ -0,0 +1,280 @@
+import DatabaseConfig from "../DatabaseConfig";
+import BaseService from "./BaseService";
+import IncomingCallLogItemService from "./IncomingCallLogItemService";
+import IncomingCallLogService from "./IncomingCallLogService";
+import IncomingCallPolicyService from "./IncomingCallPolicyService";
+import ProjectService from "./ProjectService";
+import UserNotificationSettingService from "./UserNotificationSettingService";
+import MissedCallNotification, {
+  MissedCall,
+  MissedCallAttempt,
+  MissedCallNotificationContent,
+} from "../Utils/IncomingCall/MissedCallNotification";
+import logger, { LogAttributes } from "../Utils/Logger";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import URL from "../../Types/API/URL";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
+import MissedIncomingCall, {
+  MissedIncomingCallReason,
+} from "../../Types/IncomingCall/MissedIncomingCall";
+import ObjectID from "../../Types/ObjectID";
+import IncomingCallLog from "../../Models/DatabaseModels/IncomingCallLog";
+import IncomingCallLogItem from "../../Models/DatabaseModels/IncomingCallLogItem";
+import User from "../../Models/DatabaseModels/User";
+
+/*
+ * Tells an Incoming Call Policy's owners when a call to it ended without
+ * reaching anyone (issue #4159). Before this, a missed call was only a row in
+ * the policy's call log that somebody had to go and look at.
+ *
+ * Owner users and the accepted members of owner teams are told; a policy with
+ * no owners falls back to the project owners, as monitor status changes do.
+ * Each person gets it on the channels they chose for "missed call" in
+ * Notification Settings - email unless they changed it.
+ */
+export class IncomingCallMissedCallNotificationService extends BaseService {
+  /**
+   * Never throws. It runs once the call has already ended, and nothing that
+   * goes wrong while telling people about it may reach the call provider's
+   * webhook response.
+   */
+  @CaptureSpan()
+  public async notifyOwnersOfMissedCall(data: {
+    incomingCallLogId: ObjectID;
+  }): Promise<void> {
+    try {
+      await this.notifyOwners(data.incomingCallLogId);
+    } catch (err: unknown) {
+      logger.error(
+        `IncomingCallMissedCallNotificationService: failed to notify owners about missed call ${data.incomingCallLogId?.toString()}: ${err}`,
+        {
+          incomingCallLogId: data.incomingCallLogId?.toString(),
+        } as LogAttributes,
+      );
+    }
+  }
+
+  public async getIncomingCallLogLinkInDashboard(data: {
+    projectId: ObjectID;
+    incomingCallPolicyId: ObjectID;
+    incomingCallLogId: ObjectID;
+  }): Promise<URL> {
+    const dashboardUrl: URL = await DatabaseConfig.getDashboardUrl();
+
+    /*
+     * Mirrors the Dashboard's ON_CALL_DUTY_INCOMING_CALL_POLICY_VIEW_LOG_VIEW
+     * route. Spelled out because Common/Server cannot import App sources.
+     */
+    return URL.fromString(dashboardUrl.toString()).addRoute(
+      `/${data.projectId.toString()}/on-call-duty/incoming-call-policies/${data.incomingCallPolicyId.toString()}/logs/${data.incomingCallLogId.toString()}`,
+    );
+  }
+
+  private async notifyOwners(incomingCallLogId: ObjectID): Promise<void> {
+    const callLog: IncomingCallLog | null =
+      await IncomingCallLogService.findOneById({
+        id: incomingCallLogId,
+        select: {
+          _id: true,
+          projectId: true,
+          incomingCallPolicyId: true,
+          status: true,
+          statusMessage: true,
+          callerPhoneNumber: true,
+          routingPhoneNumber: true,
+          startedAt: true,
+          endedAt: true,
+          project: {
+            name: true,
+          },
+          incomingCallPolicy: {
+            name: true,
+          },
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+    if (!callLog || !callLog.projectId || !callLog.incomingCallPolicyId) {
+      logger.error(
+        `IncomingCallMissedCallNotificationService: call log ${incomingCallLogId.toString()} was not found, so nobody can be told it was missed.`,
+      );
+      return;
+    }
+
+    const projectId: ObjectID = callLog.projectId;
+    const incomingCallPolicyId: ObjectID = callLog.incomingCallPolicyId;
+
+    const logAttributes: LogAttributes = {
+      projectId: projectId.toString(),
+      incomingCallPolicyId: incomingCallPolicyId.toString(),
+      incomingCallLogId: incomingCallLogId.toString(),
+    } as LogAttributes;
+
+    const reason: MissedIncomingCallReason | null =
+      MissedIncomingCall.getReason({
+        status: callLog.status,
+        statusMessage: callLog.statusMessage,
+      });
+
+    // Only a call that ended without reaching anyone is a missed call.
+    if (!reason) {
+      logger.debug(
+        `IncomingCallMissedCallNotificationService: call log ${incomingCallLogId.toString()} ended as ${callLog.status?.toString()}, which is not a missed call. Nobody is notified.`,
+        logAttributes,
+      );
+      return;
+    }
+
+    let isOwner: boolean = true;
+    let recipients: Array<User> =
+      await IncomingCallPolicyService.findOwners(incomingCallPolicyId);
+
+    if (recipients.length === 0) {
+      isOwner = false;
+      recipients = await ProjectService.getOwners(projectId);
+    }
+
+    if (recipients.length === 0) {
+      logger.warn(
+        `IncomingCallMissedCallNotificationService: incoming call policy ${incomingCallPolicyId.toString()} has no owners and project ${projectId.toString()} has no owners, so nobody can be told about missed call ${incomingCallLogId.toString()}. Add an owner to the policy.`,
+        logAttributes,
+      );
+      return;
+    }
+
+    const call: MissedCall = {
+      policyName:
+        callLog.incomingCallPolicy?.name?.toString() || "Incoming Call Policy",
+      projectName: callLog.project?.name?.toString() || "",
+      callerPhoneNumber: callLog.callerPhoneNumber?.toString() || "",
+      routingPhoneNumber: callLog.routingPhoneNumber?.toString() || "",
+      reason: reason,
+      statusMessage: callLog.statusMessage || "",
+      startedAt: callLog.startedAt,
+      endedAt: callLog.endedAt,
+      attempts: await this.getAttempts(incomingCallLogId),
+      incomingCallLogViewLink: (
+        await this.getIncomingCallLogLinkInDashboard({
+          projectId: projectId,
+          incomingCallPolicyId: incomingCallPolicyId,
+          incomingCallLogId: incomingCallLogId,
+        })
+      ).toString(),
+    };
+
+    const seenUserIds: Set<string> = new Set<string>();
+
+    for (const user of recipients) {
+      const userId: ObjectID | null | undefined = user.id;
+
+      if (!userId || seenUserIds.has(userId.toString())) {
+        continue;
+      }
+
+      seenUserIds.add(userId.toString());
+
+      /*
+       * Per-recipient isolation: one person with a broken notification
+       * setup must not cost the others their notification.
+       */
+      try {
+        const content: MissedCallNotificationContent =
+          MissedCallNotification.buildContent({
+            call: call,
+            isOwner: isOwner,
+            timezone: user.timezone,
+          });
+
+        /*
+         * sendUserNotification sends nothing without a setting row. Members
+         * who joined before this event existed get the default (email on)
+         * here; a row the user already switched off is left alone.
+         */
+        await UserNotificationSettingService.ensureSettingExistsForUser({
+          userId: userId,
+          projectId: projectId,
+          eventType: MissedCallNotification.EVENT_TYPE,
+        });
+
+        await UserNotificationSettingService.sendUserNotification({
+          userId: userId,
+          projectId: projectId,
+          eventType: MissedCallNotification.EVENT_TYPE,
+          emailEnvelope: content.emailEnvelope,
+          smsMessage: content.smsMessage,
+          callRequestMessage: content.callRequestMessage,
+          pushNotificationMessage: content.pushNotificationMessage,
+          whatsAppMessage: content.whatsAppMessage,
+        });
+      } catch (err: unknown) {
+        logger.error(
+          `IncomingCallMissedCallNotificationService: failed to notify user ${userId.toString()} about missed call ${incomingCallLogId.toString()}: ${err}`,
+          logAttributes,
+        );
+      }
+    }
+  }
+
+  // Every dial attempt of the call, in the order it was made.
+  private async getAttempts(
+    incomingCallLogId: ObjectID,
+  ): Promise<Array<MissedCallAttempt>> {
+    const items: Array<IncomingCallLogItem> =
+      await IncomingCallLogItemService.findBy({
+        query: {
+          incomingCallLogId: incomingCallLogId,
+        },
+        select: {
+          _id: true,
+          status: true,
+          startedAt: true,
+          endedAt: true,
+          userPhoneNumber: true,
+          user: {
+            name: true,
+            email: true,
+          },
+          incomingCallPolicyEscalationRule: {
+            name: true,
+            order: true,
+          },
+        },
+        sort: {
+          startedAt: SortOrder.Ascending,
+        },
+        skip: 0,
+        limit: LIMIT_PER_PROJECT,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    return items.map((item: IncomingCallLogItem): MissedCallAttempt => {
+      const ruleName: string =
+        item.incomingCallPolicyEscalationRule?.name?.toString().trim() || "";
+      const ruleOrder: number | undefined =
+        item.incomingCallPolicyEscalationRule?.order;
+
+      return {
+        userName:
+          item.user?.name?.toString() || item.user?.email?.toString() || "",
+        phoneNumber: item.userPhoneNumber?.toString() || "",
+        ruleName:
+          ruleName ||
+          (ruleOrder !== undefined && ruleOrder !== null
+            ? `Rule ${ruleOrder}`
+            : ""),
+        status: item.status,
+        ringSeconds: MissedCallNotification.getSecondsBetween(
+          item.startedAt,
+          item.endedAt,
+        ),
+      };
+    });
+  }
+}
+
+export default new IncomingCallMissedCallNotificationService();

package/Server/Services/IncomingCallPolicyService.ts

@@ -14,6 +14,14 @@ import BadDataException from "../../Types/Exception/BadDataException";
 import QueryHelper from "../Types/Database/QueryHelper";
 import ObjectID from "../../Types/ObjectID";
 import ModelPermission from "../Types/Database/Permissions/Index";
+import IncomingCallPolicyOwnerTeamService from "./IncomingCallPolicyOwnerTeamService";
+import IncomingCallPolicyOwnerUserService from "./IncomingCallPolicyOwnerUserService";
+import TeamMemberService from "./TeamMemberService";
+import IncomingCallPolicyOwnerTeam from "../../Models/DatabaseModels/IncomingCallPolicyOwnerTeam";
+import IncomingCallPolicyOwnerUser from "../../Models/DatabaseModels/IncomingCallPolicyOwnerUser";
+import User from "../../Models/DatabaseModels/User";
+import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
+import Select from "../Types/Database/Select";
 
 function normalizeProjectCallSMSConfigIdValue(
   value: unknown,
@@ -323,6 +331,116 @@ export class Service extends DatabaseService<IncomingCallPolicy> {
     };
   }
 
+  /*
+   * The people who own this policy: its owner users, plus the accepted
+   * members of its owner teams, once each. Owners who are no longer accepted
+   * members of the project are left out. Empty means nobody owns the policy,
+   * which callers treat as "tell the project owners".
+   */
+  @CaptureSpan()
+  public async findOwners(
+    incomingCallPolicyId: ObjectID,
+  ): Promise<Array<User>> {
+    if (!incomingCallPolicyId) {
+      throw new BadDataException("incomingCallPolicyId is required");
+    }
+
+    const ownerUsers: Array<IncomingCallPolicyOwnerUser> =
+      await IncomingCallPolicyOwnerUserService.findBy({
+        query: {
+          incomingCallPolicyId: incomingCallPolicyId,
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          user: {
+            _id: true,
+            email: true,
+            name: true,
+            timezone: true,
+          } as Select<User>,
+        },
+        skip: 0,
+        limit: LIMIT_PER_PROJECT,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    const ownerTeams: Array<IncomingCallPolicyOwnerTeam> =
+      await IncomingCallPolicyOwnerTeamService.findBy({
+        query: {
+          incomingCallPolicyId: incomingCallPolicyId,
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          teamId: true,
+        },
+        skip: 0,
+        limit: LIMIT_PER_PROJECT,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    const users: Array<User> = [];
+    const seenUserIds: Set<string> = new Set<string>();
+
+    const addUser: (user: User | undefined) => void = (
+      user: User | undefined,
+    ): void => {
+      const userId: string | undefined = user?.id?.toString();
+
+      if (!user || !userId || seenUserIds.has(userId)) {
+        return;
+      }
+
+      seenUserIds.add(userId);
+      users.push(user);
+    };
+
+    for (const ownerUser of ownerUsers) {
+      addUser(ownerUser.user);
+    }
+
+    const teamIds: Array<ObjectID> = ownerTeams
+      .map((ownerTeam: IncomingCallPolicyOwnerTeam): ObjectID | undefined => {
+        return ownerTeam.teamId;
+      })
+      .filter((teamId: ObjectID | undefined): teamId is ObjectID => {
+        return Boolean(teamId);
+      });
+
+    if (teamIds.length > 0) {
+      /*
+       * Accepted rows only: a pending invitation to an owner team grants
+       * none of the team's permissions, so it does not make anyone an owner.
+       */
+      const teamUsers: Array<User> = await TeamMemberService.getUsersInTeams(
+        teamIds,
+        { acceptedOnly: true },
+      );
+
+      for (const teamUser of teamUsers) {
+        addUser(teamUser);
+      }
+    }
+
+    const projectId: ObjectID | undefined =
+      ownerUsers[0]?.projectId || ownerTeams[0]?.projectId;
+
+    if (!projectId) {
+      return [];
+    }
+
+    // Owners who left the project are not told about its calls.
+    return await TeamMemberService.filterUsersToProjectMembers({
+      projectId: projectId,
+      users: users,
+    });
+  }
+
   @CaptureSpan()
   protected override async onCreateSuccess(
     _onCreate: OnCreate<IncomingCallPolicy>,

package/Server/Services/Index.ts

@@ -436,6 +436,7 @@ import HostOwnerRuleService from "./HostOwnerRuleService";
 import HostOwnerTeamService from "./HostOwnerTeamService";
 import HostOwnerUserService from "./HostOwnerUserService";
 import IncomingCallPolicyService from "./IncomingCallPolicyService";
+import IncomingCallLogService from "./IncomingCallLogService";
 import IncomingCallPolicyPhoneNumberService from "./IncomingCallPolicyPhoneNumberService";
 import PushNotificationLogService from "./PushNotificationLogService";
 import RunnerOwnerTeamService from "./RunnerOwnerTeamService";
@@ -460,6 +461,9 @@ import IoTFleetOwnerUserService from "./IoTFleetOwnerUserService";
 import UserNotificationEmailRollupItemService from "./UserNotificationEmailRollupItemService";
 import UserNotificationEmailRollupBatchService from "./UserNotificationEmailRollupBatchService";
 import UserNotificationEmailRollupSettingService from "./UserNotificationEmailRollupSettingService";
+import McpOAuthClientService from "./McpOAuthClientService";
+import McpOAuthGrantService from "./McpOAuthGrantService";
+import McpOAuthTokenService from "./McpOAuthTokenService";
 
 const services: Array<BaseService> = [
   OnCallDutyPolicyTimeLogService,
@@ -903,6 +907,7 @@ const services: Array<BaseService> = [
   IncomingCallPolicyOwnerRuleService,
   IncomingCallPolicyOwnerTeamService,
   IncomingCallPolicyOwnerUserService,
+  IncomingCallLogService,
   IoTFleetLabelRuleService,
   IoTFleetOwnerRuleService,
   IoTFleetOwnerTeamService,
@@ -918,6 +923,9 @@ const services: Array<BaseService> = [
   UserNotificationEmailRollupItemService,
   UserNotificationEmailRollupBatchService,
   UserNotificationEmailRollupSettingService,
+  McpOAuthClientService,
+  McpOAuthGrantService,
+  McpOAuthTokenService,
 ];
 
 export const AnalyticsServices: Array<

package/Server/Services/KubernetesClusterAiAccessService.ts

@@ -104,9 +104,11 @@ export {
  */
 
 export interface KubernetesClusterAiAccessProjectGates {
+  /*
+   * Project.enableAi, the project's only AI switch: it covers
+   * investigation, fixes and AI commands on Runners alike.
+   */
   isAiEnabled: boolean;
-  isAutoRemediationEnabled: boolean;
-  isAiCommandExecutionEnabled: boolean;
   hasLlmProvider: boolean;
   /*
    * Why no AI run can start for lack of AI credits (AIService.
@@ -458,7 +460,7 @@ export function getPreviousInstanceRetryAfterSeconds(data: {
 
 /*
  * One thing an agent Runner row holds beyond what registration gave it, and
- * what an operator does under Project Settings → Runners to remove it.
+ * what an operator does under Runbooks → Runners to remove it.
  */
 export interface AgentRunnerHolding {
   description: string;
@@ -524,8 +526,6 @@ class KubernetesClusterAiAccessServiceClass {
       id: projectId,
       select: {
         enableAi: true,
-        enableAutoRemediation: true,
-        enableAiCommandExecution: true,
         enableAutomaticIncidentInvestigation: true,
         enableAutomaticAlertInvestigation: true,
       },
@@ -566,11 +566,8 @@ class KubernetesClusterAiAccessServiceClass {
     }
 
     return {
-      // Kill switches: a missing column counts as enabled (=== false idiom).
+      // Kill switch: a missing column counts as enabled (=== false idiom).
       isAiEnabled: project?.enableAi !== false,
-      isAutoRemediationEnabled: project?.enableAutoRemediation !== false,
-      // Explicit opt-in (=== true idiom), same as the Bash/SSH lane.
-      isAiCommandExecutionEnabled: project?.enableAiCommandExecution === true,
       // A failed lookup reads as none, like no provider at all.
       hasLlmProvider: Boolean(llmProvider),
       aiBalanceBlocker,
@@ -878,40 +875,6 @@ class KubernetesClusterAiAccessServiceClass {
       });
     }
 
-    if (!gates.isAutoRemediationEnabled) {
-      gaps.push({
-        code: "project_auto_remediation_disabled",
-        title: "Auto-remediation is disabled for this project",
-        description:
-          "The project-level auto-remediation kill switch is off, so no AI fix can be proposed or run.",
-        nextStep:
-          "Enable auto-remediation under Project Settings → AI Features.",
-        blocks: "remediation",
-      });
-    }
-
-    /*
-     * The project's command-execution opt-in governs AI-composed commands
-     * on Runners. A cluster reached through its Kubernetes AI agent (or the
-     * chart's previous in-cluster Runner) is governed by the cluster's own
-     * Fixes setting and the chart's write RBAC instead, so the opt-in only
-     * gates a cluster reached through an advanced Runner and credential.
-     */
-    if (
-      loaded.target.type === "advanced_runner" &&
-      !gates.isAiCommandExecutionEnabled
-    ) {
-      gaps.push({
-        code: "project_ai_command_execution_disabled",
-        title: "AI command execution is not enabled for this project",
-        description:
-          "This cluster is reached through a Runner, and AI-composed commands (kubectl fixes included) never run on a Runner in a project that has not opted in.",
-        nextStep:
-          'Turn on "Enable AI Command Execution" under Project Settings → AI Features.',
-        blocks: "remediation",
-      });
-    }
-
     const blocksInvestigation: boolean = gaps.some(
       (gap: KubernetesAiAccessGap) => {
         return gap.blocks === "investigation" || gap.blocks === "both";
@@ -1201,7 +1164,7 @@ class KubernetesClusterAiAccessServiceClass {
         title: "The Runner does not accept AI commands",
         description: `"Runs AI Remediation Commands" is turned off on Runner "${runner.name}", so it will not be served kubectl work.`,
         nextStep:
-          'Turn on "Runs AI Remediation Commands" on the Runner (Project Settings → Runners).',
+          'Turn on "Runs AI Remediation Commands" on the Runner (Runbooks → Runners).',
         blocks: "both",
       });
     }
@@ -1270,7 +1233,7 @@ class KubernetesClusterAiAccessServiceClass {
             : credential.credentialType !== RunbookCredentialType.Kubernetes
               ? `"${credential.name}" is not a Kubernetes credential.`
               : `"${credential.name}" is not assigned to Runner "${runner.name}".`,
-          nextStep: `Create a Kubernetes credential (API server URL + ServiceAccount token) under Project Settings → Runner Credentials, assign it to the Runner, and select it on ${CLUSTER_AI_AGENT_PAGE}.`,
+          nextStep: `Create a Kubernetes credential (API server URL + ServiceAccount token) under Runbooks → Runner Credentials, assign it to the Runner, and select it on ${CLUSTER_AI_AGENT_PAGE}.`,
           blocks: "both",
         });
       } else {
@@ -2221,7 +2184,7 @@ class KubernetesClusterAiAccessServiceClass {
        */
       throw new KubernetesAgentRegistrationRefusedException({
         reason: "runner_holds_more_than_defaults",
-        message: `Upgrade the Kubernetes agent chart: its Kubernetes AI agent replaces this in-cluster Runner. Or, under Project Settings → Runners, on Runner "${data.runner.name}": ${this.describeHoldingRemedies(
+        message: `Upgrade the Kubernetes agent chart: its Kubernetes AI agent replaces this in-cluster Runner. Or, under Runbooks → Runners, on Runner "${data.runner.name}": ${this.describeHoldingRemedies(
           holdings,
         )}. It is offline, but it holds more than an in-cluster Runner's defaults (${this.describeHoldings(
           holdings,
@@ -2351,7 +2314,7 @@ class KubernetesClusterAiAccessServiceClass {
       );
 
       throw new TooManyRequestsException(
-        `This project already has ${totalAgentRunners} in-cluster Runners, which is its limit (${MAX_KUBERNETES_AGENT_RUNNERS_PER_PROJECT}). Delete the Runners of clusters that no longer exist under Project Settings → Runners, then retry.`,
+        `This project already has ${totalAgentRunners} in-cluster Runners, which is its limit (${MAX_KUBERNETES_AGENT_RUNNERS_PER_PROJECT}). Delete the Runners of clusters that no longer exist under Runbooks → Runners, then retry.`,
       );
     }
 

package/Server/Services/KubernetesClusterService.ts

@@ -1085,7 +1085,7 @@ export class Service extends DatabaseService<Model> {
 
       if (isAgentRunner && !isThisClustersAgent && isRunnerChanged) {
         throw new BadDataException(
-          `Runner "${runner.name}" is the in-cluster Runner the Kubernetes agent chart installed on another cluster. Its ServiceAccount reaches only the cluster its pod runs in, and it is never given a credential, so it cannot run kubectl for this cluster. Leave the Runner empty to use this cluster's Kubernetes AI agent, or bind a Runner you created under Project Settings → Runners together with a Kubernetes credential for this cluster.`,
+          `Runner "${runner.name}" is the in-cluster Runner the Kubernetes agent chart installed on another cluster. Its ServiceAccount reaches only the cluster its pod runs in, and it is never given a credential, so it cannot run kubectl for this cluster. Leave the Runner empty to use this cluster's Kubernetes AI agent, or bind a Runner you created under Runbooks → Runners together with a Kubernetes credential for this cluster.`,
         );
       }
 
@@ -1095,7 +1095,7 @@ export class Service extends DatabaseService<Model> {
 
       if (isAgentRunner) {
         throw new BadDataException(
-          `Runner "${runner.name}" is an in-cluster Runner the Kubernetes agent chart installed: it runs kubectl with its own ServiceAccount only and is never given a credential. Clear the credential, or bind a Runner you created under Project Settings → Runners that the credential is assigned to.`,
+          `Runner "${runner.name}" is an in-cluster Runner the Kubernetes agent chart installed: it runs kubectl with its own ServiceAccount only and is never given a credential. Clear the credential, or bind a Runner you created under Runbooks → Runners that the credential is assigned to.`,
         );
       }
 
@@ -1121,7 +1121,7 @@ export class Service extends DatabaseService<Model> {
 
       if (!isAssigned) {
         throw new BadDataException(
-          `The Kubernetes credential is not assigned to Runner "${runner.name}", so that Runner could never use it for this cluster (a credential is handed only to the Runners it is assigned to). Assign it to the Runner under Project Settings → Runner Credentials first, or choose a credential that is assigned to it.`,
+          `The Kubernetes credential is not assigned to Runner "${runner.name}", so that Runner could never use it for this cluster (a credential is handed only to the Runners it is assigned to). Assign it to the Runner under Runbooks → Runner Credentials first, or choose a credential that is assigned to it.`,
         );
       }
     }

package/Server/Services/LogAggregationService.ts

@@ -282,12 +282,13 @@ export class LogAggregationService {
   /*
    * Read-side retention filter (mirrors
    * AnalyticsDatabaseService.getRetentionReadFilter): rows past their
-   * per-service retention stay queryable until their whole part drops
-   * (ttl_only_drop_parts), so raw-table reads exclude them explicitly.
+   * retention stay on disk until the midnight after it (Log's TTL is
+   * rounded up to the day, see RetentionTtl), so raw-table reads exclude
+   * them explicitly.
    * Deliberately NOT applied to projection-shaped queries (the severity
-   * histogram): an aggregate projection cannot evaluate a predicate on a
-   * column it does not store, so adding it would silently force a full
-   * base-table scan.
+   * histogram with nothing but severities to filter on): an aggregate
+   * projection cannot evaluate a predicate on a column it does not store,
+   * so adding it would silently force a full base-table scan.
    */
   private static readonly RETENTION_FILTER: string =
     " AND retentionDate >= now()";
@@ -399,6 +400,20 @@ export class LogAggregationService {
 
     LogAggregationService.appendCommonFilters(statement, request);
 
+    /*
+     * Where it is free, the histogram leaves out rows past their retention
+     * like every other read: a filter the projection does not store already
+     * sends this query to the base table, and the retention filter only adds
+     * a column to that scan. On the projection-only shape it would force the
+     * scan, so there it is left out - and a window reaching back past the
+     * retention can count, at its far edge, up to a day of rows the list no
+     * longer shows (they are on disk until the midnight after their
+     * retentionDate).
+     */
+    if (LogAggregationService.histogramScansBaseTable(request)) {
+      statement.append(LogAggregationService.RETENTION_FILTER);
+    }
+
     statement.append(
       " GROUP BY minute, severityText ) GROUP BY bucket, severityText ORDER BY bucket ASC",
     );
@@ -421,6 +436,24 @@ export class LogAggregationService {
     return statement;
   }
 
+  /*
+   * Whether a histogram request filters on anything proj_severity_histogram
+   * does not store (it keeps projectId, severityText and the minute) - that
+   * is, whether its query reads the base table anyway. Decided by what the
+   * filters render, not by which fields are set: an empty attribute list or
+   * resource scope renders nothing and leaves the projection in use.
+   */
+  private static histogramScansBaseTable(request: HistogramRequest): boolean {
+    const filtersBeyondSeverity: Statement = new Statement();
+
+    LogAggregationService.appendCommonFilters(filtersBeyondSeverity, {
+      ...request,
+      severityTexts: undefined,
+    });
+
+    return filtersBeyondSeverity.query.trim().length > 0;
+  }
+
   private static buildFacetStatement(request: FacetRequest): Statement {
     // Pre-rename alias from stale clients; the V3 column is primaryEntityId.
     if (request.facetKey === "serviceId") {