@oneuptime/common 12.0.31 → 12.0.33

package/Tests/Server/Utils/AI/ObservabilityChatPrompt.test.ts

@@ -4,8 +4,25 @@ import {
 } from "../../../../Server/Utils/AI/Chat/ObservabilityChatPrompt";
 import AIChatPageContextType, {
   AIChatPageContext,
+  AIChatPageContextHelper,
 } from "../../../../Types/AI/AIChatPageContext";
 import AIChatPermissionMode from "../../../../Types/AI/AIChatPermissionMode";
+/*
+ * Toolbox/Index MUST be imported before any individual tool module. The
+ * toolbox sits in an import cycle (tool -> service -> ... -> AIToolbox), so
+ * entering through a tool module first re-enters Index before its tools array
+ * is assigned and leaves undefined holes in it. Production always enters
+ * through Index, which is the order reproduced here.
+ */
+import AIToolbox from "../../../../Server/Utils/AI/Toolbox/Index";
+import {
+  QueryIncidentsTool,
+  SearchIncidentsTool,
+} from "../../../../Server/Utils/AI/Toolbox/IncidentTools";
+import { QueryAlertsTool } from "../../../../Server/Utils/AI/Toolbox/AlertTools";
+import { QueryMonitorsTool } from "../../../../Server/Utils/AI/Toolbox/MonitorTools";
+import { ObservabilityTool } from "../../../../Server/Utils/AI/Toolbox/ToolTypes";
+import { JSONObject } from "../../../../Types/JSON";
 import { describe, expect, test } from "@jest/globals";
 
 const ENTITY_ID: string = "0f10509d-6656-4a08-b957-235fd4e8c52e";
@@ -102,6 +119,58 @@ describe("buildPageContextSection", () => {
     },
   );
 
+  /*
+   * Regression for #3552. The incidents list ships a suggested prompt card
+   * asking "which incidents are currently active or unresolved, and what
+   * state is each in?" — and the model replied that it had no tool for that
+   * and could only act on one incident once given an incidentId. It was not
+   * hallucinating: this guidance summarised the tool as "query_incidents
+   * (recent incidents, or one by incidentId)", a NARROWER capability claim
+   * than query_incidents' own schema, injected on that exact page. The model
+   * paraphrased the parenthetical back as a denial. Page guidance must name
+   * the affordance the page's own suggested prompts depend on.
+   */
+  test('incidents list guidance offers state="active" for unresolved incidents', () => {
+    const section: string = buildPageContextSection({
+      type: AIChatPageContextType.IncidentsList,
+    });
+
+    expect(section).toContain('state="active"');
+    expect(section).toContain("unresolved");
+    // The exact phrasing the model quoted back as "I have no such tool".
+    expect(section).not.toContain("(recent incidents, or one by incidentId)");
+  });
+
+  /*
+   * The alerts list carries the same "what is firing right now?" suggested
+   * prompt, and query_alerts has the same state="active" affordance. Its
+   * guidance used to stop dead at the tool name ("answered with
+   * query_alerts."), which is the same under-claim one page over.
+   */
+  test('alerts list guidance offers state="active" for unresolved alerts', () => {
+    const section: string = buildPageContextSection({
+      type: AIChatPageContextType.AlertsList,
+    });
+
+    expect(section).toContain('state="active"');
+    expect(section).toContain("unresolved");
+    expect(section).not.toContain("answered with query_alerts.");
+  });
+
+  /*
+   * "What is down right now?" on the monitors list is query_monitors'
+   * problemsOnly flag. Naming only the tool left the model to guess that a
+   * plain listing was the best it could do.
+   */
+  test("monitors list guidance offers problemsOnly for what is down now", () => {
+    const section: string = buildPageContextSection({
+      type: AIChatPageContextType.MonitorsList,
+    });
+
+    expect(section).toContain("problemsOnly");
+    expect(section).not.toContain("answered with query_monitors.");
+  });
+
   test("every section reminds the model that context is not evidence", () => {
     const section: string = buildPageContextSection({
       type: AIChatPageContextType.Incident,
@@ -168,6 +237,26 @@ describe("buildObservabilityChatSystemPrompt with page context", () => {
     expect(prompt).toContain("## Hard rules");
     expect(prompt).toContain("Cite your sources.");
   });
+
+  /*
+   * buildPageContextSection passing is worthless if the section never makes
+   * it into the prompt: the assembled string is what actually reaches the
+   * model. #3552 was reported by a user sitting on the incidents list, so
+   * that context is asserted end to end.
+   */
+  test('an incidents list context carries state="active" into the prompt', () => {
+    const prompt: string = buildObservabilityChatSystemPrompt({
+      currentTime: new Date("2026-07-16T00:00:00Z"),
+      permissionMode: AIChatPermissionMode.ReadOnly,
+      pageContext: {
+        type: AIChatPageContextType.IncidentsList,
+      },
+    });
+
+    expect(prompt).toContain("## Current page context");
+    expect(prompt).toContain('state="active"');
+    expect(prompt).not.toContain("(recent incidents, or one by incidentId)");
+  });
 });
 
 /*
@@ -344,3 +433,167 @@ describe("buildObservabilityChatSystemPrompt — action guidance per mode", () =
     },
   );
 });
+
+/*
+ * The page guidance is a SUMMARY of tools the model also receives in full.
+ * When the summary claims LESS than the schema, the model believes the
+ * summary — that is #3552. These tests assert both halves at once: that the
+ * tool really does take the argument, and that the page whose suggested
+ * prompts depend on it says so. Adding an area page means adding a row here,
+ * or deciding explicitly that the page has no "what is wrong right now?"
+ * affordance to advertise.
+ *
+ * A type alias rather than an interface on purpose: jest's object-form
+ * test.each is typed against Record<string, unknown>, which an interface does
+ * not satisfy (it has no implicit index signature).
+ */
+type AreaAffordanceCase = {
+  type: AIChatPageContextType;
+  tool: ObservabilityTool;
+  // The tool argument that answers "what is wrong RIGHT NOW?" on this page.
+  argument: string;
+  // The enum value the model must pass, or null when the argument is a flag.
+  enumValue: string | null;
+  // The literal text the page guidance must use to name the affordance.
+  guidancePhrase: string;
+};
+
+function getSchemaProperty(
+  tool: ObservabilityTool,
+  argument: string,
+): JSONObject | undefined {
+  const properties: JSONObject =
+    (tool.inputSchema["properties"] as JSONObject | undefined) || {};
+
+  return properties[argument] as JSONObject | undefined;
+}
+
+describe("page guidance stays in sync with the tool schemas it summarises", () => {
+  test.each<AreaAffordanceCase>([
+    {
+      type: AIChatPageContextType.IncidentsList,
+      tool: QueryIncidentsTool,
+      argument: "state",
+      enumValue: "active",
+      guidancePhrase: 'state="active"',
+    },
+    {
+      type: AIChatPageContextType.AlertsList,
+      tool: QueryAlertsTool,
+      argument: "state",
+      enumValue: "active",
+      guidancePhrase: 'state="active"',
+    },
+    {
+      type: AIChatPageContextType.MonitorsList,
+      tool: QueryMonitorsTool,
+      argument: "problemsOnly",
+      enumValue: null,
+      guidancePhrase: "problemsOnly",
+    },
+  ])(
+    "$type guidance names the $argument affordance its tool really has",
+    (testCase: AreaAffordanceCase) => {
+      // Half one: the tool's own schema documents the affordance.
+      const property: JSONObject | undefined = getSchemaProperty(
+        testCase.tool,
+        testCase.argument,
+      );
+
+      expect(property).toBeDefined();
+
+      if (testCase.enumValue) {
+        const allowedValues: Array<string> =
+          (property?.["enum"] as Array<string> | undefined) || [];
+
+        expect(allowedValues).toContain(testCase.enumValue);
+      }
+
+      // Half two: the page the user is standing on is told about it.
+      const section: string = buildPageContextSection({ type: testCase.type });
+
+      expect(section).toContain(testCase.tool.name);
+      expect(section).toContain(testCase.guidancePhrase);
+    },
+  );
+
+  /*
+   * The incidents list owns the historical question too, and search_incidents
+   * is the only tool that searches incident text. Dropping it here would push
+   * "have we seen this before?" onto query_incidents, which cannot search at
+   * all. This one is a drift guard: it holds before and after the fix.
+   */
+  test("the incidents list still routes free-text history to search_incidents", () => {
+    const section: string = buildPageContextSection({
+      type: AIChatPageContextType.IncidentsList,
+    });
+
+    expect(getSchemaProperty(SearchIncidentsTool, "searchText")).toBeDefined();
+    expect(section).toContain(SearchIncidentsTool.name);
+  });
+});
+
+/*
+ * The other direction of the same drift: guidance naming a tool the model was
+ * never given. Extracting tool names from prose is safe here because this
+ * prompt keeps a strict convention — tool names are snake_case, tool
+ * ARGUMENTS are camelCase (incidentId, problemsOnly, createdWithinHours), and
+ * everything else is plain English or a UUID. So a snake_case token in the
+ * guidance is always a capability claim, and every one of them is checked
+ * against the toolbox the runner actually offers. A fresh regex per call
+ * keeps the global flag's lastIndex out of the picture.
+ */
+function extractToolNameTokens(text: string): Array<string> {
+  const tokens: Array<string> =
+    text.match(/[a-z][a-z0-9]*(?:_[a-z0-9]+)+/g) || [];
+
+  return Array.from(new Set(tokens));
+}
+
+const TOOLBOX_TOOL_NAMES: Set<string> = new Set(
+  AIToolbox.getTools().map((tool: ObservabilityTool): string => {
+    return tool.name;
+  }),
+);
+
+function buildSectionForType(type: AIChatPageContextType): string {
+  if (!AIChatPageContextHelper.isEntityType(type)) {
+    return buildPageContextSection({ type: type });
+  }
+
+  return buildPageContextSection({
+    type: type,
+    entityId: type === AIChatPageContextType.Trace ? TRACE_ID : ENTITY_ID,
+  });
+}
+
+describe("page guidance only names tools that exist in the toolbox", () => {
+  /*
+   * Driven off the enum rather than a hand-written list, so a new page type
+   * fails here until someone decides what it routes to.
+   */
+  test.each<[AIChatPageContextType]>(
+    Object.values(AIChatPageContextType).map(
+      (type: AIChatPageContextType): [AIChatPageContextType] => {
+        return [type];
+      },
+    ),
+  )(
+    "%s names at least one tool and every tool it names is real",
+    (type: AIChatPageContextType) => {
+      const section: string = buildSectionForType(type);
+      const mentionedTools: Array<string> = extractToolNameTokens(section);
+
+      // Guidance that routes the model nowhere is guidance it cannot use.
+      expect(mentionedTools.length).toBeGreaterThan(0);
+
+      const notInToolbox: Array<string> = mentionedTools.filter(
+        (name: string): boolean => {
+          return !TOOLBOX_TOOL_NAMES.has(name);
+        },
+      );
+
+      expect(notInToolbox).toEqual([]);
+    },
+  );
+});

package/Tests/Server/Utils/BasicCron.test.ts

@@ -0,0 +1,283 @@
+import { beforeEach, describe, expect, jest, test } from "@jest/globals";
+import cron from "node-cron";
+import logger from "../../../Server/Utils/Logger";
+import BasicCron from "../../../Server/Utils/BasicCron";
+
+/*
+ * The wrapper every scheduled job in the app is registered through.
+ *
+ * Its job is not scheduling — node-cron does that — it is making sure one bad
+ * run cannot end the schedule. A cron callback that throws takes its timer
+ * with it, so a job that failed once at 03:00 would simply never run again,
+ * and nothing would say so: the next morning looks exactly like a night with
+ * nothing to do. That is why the tick is wrapped, and it is the behaviour
+ * worth pinning here.
+ *
+ * The startup run is deliberately NOT inside that guard, and the asymmetry is
+ * the interesting part of this module — see the last block.
+ */
+
+jest.mock("node-cron", () => {
+  return { __esModule: true, default: { schedule: jest.fn() } };
+});
+
+jest.mock("../../../Server/Utils/Logger", () => {
+  return {
+    __esModule: true,
+    default: {
+      debug: jest.fn(),
+      info: jest.fn(),
+      warn: jest.fn(),
+      error: jest.fn(),
+    },
+  };
+});
+
+const mockedCron: { schedule: jest.Mock } = cron as unknown as {
+  schedule: jest.Mock;
+};
+
+const mockedLogger: { debug: jest.Mock; error: jest.Mock } =
+  logger as unknown as { debug: jest.Mock; error: jest.Mock };
+
+const EVERY_FIVE_MINUTES: string = "*/5 * * * *";
+
+/*
+ * The run function BasicCron is handed. jest.Mock's first type parameter is
+ * the RETURN type, so this is "a mock taking no arguments and answering with
+ * a promise" - named once here rather than restated at every call site.
+ */
+type RunMock = jest.Mock<Promise<void>, []>;
+
+function succeedingRun(): RunMock {
+  const mock: RunMock = jest.fn() as unknown as RunMock;
+  mock.mockImplementation((): Promise<void> => {
+    return Promise.resolve();
+  });
+  return mock;
+}
+
+function failingRun(message: string): RunMock {
+  const mock: RunMock = jest.fn() as unknown as RunMock;
+  mock.mockImplementation((): Promise<void> => {
+    return Promise.reject(new Error(message));
+  });
+  return mock;
+}
+
+/*
+ * BasicCron is declared as returning void but its implementation is async, so
+ * every call site here goes through this. That mismatch is not incidental
+ * tidying - it is the reason a startup failure reaches callers as an
+ * unhandled rejection rather than a thrown error, which the last test covers.
+ */
+function startCron(data: {
+  jobName: string;
+  schedule: string;
+  runOnStartup: boolean;
+  runFunction: RunMock;
+}): Promise<void> {
+  return BasicCron({
+    jobName: data.jobName,
+    options: {
+      schedule: data.schedule,
+      runOnStartup: data.runOnStartup,
+    },
+    runFunction: data.runFunction as never,
+  }) as unknown as Promise<void>;
+}
+
+/*
+ * The callback node-cron was handed. Invoking it is what "the schedule fired"
+ * means, without waiting five real minutes for it.
+ */
+function scheduledTick(): () => Promise<void> {
+  return mockedCron.schedule.mock.calls[0]![1] as () => Promise<void>;
+}
+
+describe("BasicCron", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe("registration", () => {
+    test("registers the job on the schedule it was given", async () => {
+      const run: RunMock = succeedingRun();
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: false,
+        runFunction: run,
+      });
+
+      expect(mockedCron.schedule).toHaveBeenCalledTimes(1);
+      expect(mockedCron.schedule.mock.calls[0]![0]).toBe(EVERY_FIVE_MINUTES);
+    });
+
+    /*
+     * Registering must not run the job. A job that ran once per process start
+     * would fire on every rolling restart and every replica.
+     */
+    test("registering alone does not run the job", async () => {
+      const run: RunMock = succeedingRun();
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: false,
+        runFunction: run,
+      });
+
+      expect(run).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("a scheduled run", () => {
+    test("runs the job when the schedule fires", async () => {
+      const run: RunMock = succeedingRun();
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: false,
+        runFunction: run,
+      });
+
+      await scheduledTick()();
+
+      expect(run).toHaveBeenCalledTimes(1);
+    });
+
+    /*
+     * The whole point of the wrapper. An unhandled throw inside a node-cron
+     * callback ends that timer, so the job silently stops running for the
+     * lifetime of the process - and a job that never runs looks identical to
+     * a job with nothing to do.
+     */
+    test("a throwing run is swallowed, so the schedule survives it", async () => {
+      const run: RunMock = failingRun("job blew up");
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: false,
+        runFunction: run,
+      });
+
+      await expect(scheduledTick()()).resolves.toBeUndefined();
+      expect(mockedLogger.error).toHaveBeenCalled();
+    });
+
+    test("a failed run does not stop later runs from firing", async () => {
+      const run: RunMock = succeedingRun();
+      run.mockImplementationOnce((): Promise<void> => {
+        return Promise.reject(new Error("first run blew up"));
+      });
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: false,
+        runFunction: run,
+      });
+
+      await scheduledTick()();
+      await scheduledTick()();
+
+      expect(run).toHaveBeenCalledTimes(2);
+    });
+
+    /*
+     * The job's name is what makes a line in the log attributable to one of
+     * many scheduled jobs. Logged without it, a failure says only that
+     * something somewhere failed.
+     */
+    test("the job's name is in what it logs", async () => {
+      const run: RunMock = succeedingRun();
+
+      await startCron({
+        jobName: "DistinctiveJobName",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: false,
+        runFunction: run,
+      });
+
+      await scheduledTick()();
+
+      const logged: string = mockedLogger.debug.mock.calls
+        .map((callArgs: Array<unknown>): string => {
+          return String(callArgs[0]);
+        })
+        .join("\n");
+
+      expect(logged).toContain("DistinctiveJobName");
+    });
+  });
+
+  describe("the startup run", () => {
+    test("runs the job immediately when asked to", async () => {
+      const run: RunMock = succeedingRun();
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: true,
+        runFunction: run,
+      });
+
+      expect(run).toHaveBeenCalledTimes(1);
+    });
+
+    /*
+     * The startup run is in addition to the schedule, not instead of it - a
+     * job that ran once at boot and was never registered would go quiet after
+     * the first tick.
+     */
+    test("it still registers the recurring schedule", async () => {
+      const run: RunMock = succeedingRun();
+
+      await startCron({
+        jobName: "TestJob",
+        schedule: EVERY_FIVE_MINUTES,
+        runOnStartup: true,
+        runFunction: run,
+      });
+
+      expect(mockedCron.schedule).toHaveBeenCalledTimes(1);
+
+      await scheduledTick()();
+
+      expect(run).toHaveBeenCalledTimes(2);
+    });
+
+    /*
+     * DELIBERATE ASYMMETRY, pinned so a later reader does not "fix" it by
+     * reflex: unlike a scheduled tick, a startup failure is NOT swallowed. It
+     * rejects out of BasicCron.
+     *
+     * That is the right way round. A scheduled run fails against a live
+     * system and the next tick is a genuine retry, so surviving it is what
+     * keeps the job alive. A startup run fails against a process that is
+     * still coming up - usually because configuration or a dependency is
+     * wrong - and a boot that swallowed that would come up looking healthy
+     * with a job that never worked.
+     *
+     * Note the callers' side of this: BasicCronFunction is typed as returning
+     * void, so a caller that does not await gets an unhandled rejection
+     * rather than a thrown error. Loud either way, which is the intent.
+     */
+    test("a startup failure propagates rather than being swallowed", async () => {
+      const run: RunMock = failingRun("bad configuration at boot");
+
+      await expect(
+        startCron({
+          jobName: "TestJob",
+          schedule: EVERY_FIVE_MINUTES,
+          runOnStartup: true,
+          runFunction: run,
+        }),
+      ).rejects.toThrow("bad configuration at boot");
+    });
+  });
+});