@agent-native/toolkit 0.198.2 → 0.198.3-nightly-20260930124743

package/src/app/integrations/IntegrationsPanel.spec.tsx

@@ -22,7 +22,7 @@ vi.mock("../resources/McpIntegrationDialog.js", () => ({
   McpIntegrationDialog: () => null,
 }));
 
-vi.mock("../resources/mcp-integration-catalog.js", () => ({
+vi.mock("@agent-native/core/client/resources/mcp-integration-catalog", () => ({
   isMcpIntegrationCatalogAvailable: () => false,
   getDefaultMcpIntegrations: () => [
     {
@@ -54,11 +54,14 @@ vi.mock("../resources/mcp-integration-catalog.js", () => ({
   ],
 }));
 
-vi.mock("../resources/use-mcp-servers.js", () => mcpMocks);
+vi.mock("@agent-native/core/client/resources/use-mcp-servers", () => mcpMocks);
 
-vi.mock("./useIntegrationStatus.js", () => integrationMocks);
+vi.mock(
+  "@agent-native/core/client/integrations/useIntegrationStatus",
+  () => integrationMocks,
+);
 
-vi.mock("../i18n.js", () => ({
+vi.mock("@agent-native/core/client/i18n", () => ({
   useT: () => (key: string, options?: Record<string, unknown>) => {
     const messages: Record<string, string> = {
       "mcpIntegrations.connectionError": "Connection error",

package/src/app/observability/ObservabilityDashboard.review.spec.tsx

@@ -12,7 +12,6 @@ const {
   mockSubmitFeedback,
   mockSaveInstructionUpdate,
   mockSendToAgentChat,
-  mockTraces,
   mockTraceDetail,
   mockOpenThread,
   mockUseActionQuery,
@@ -24,7 +23,6 @@ const {
   mockSubmitFeedback: vi.fn(),
   mockSaveInstructionUpdate: vi.fn(),
   mockSendToAgentChat: vi.fn(),
-  mockTraces: vi.fn(),
   mockTraceDetail: vi.fn(),
   mockOpenThread: vi.fn(),
   mockUseActionQuery: vi.fn(),
@@ -76,7 +74,6 @@ vi.mock("./useObservability.js", () => ({
     },
     isLoading: false,
   }),
-  useTraces: (...args: unknown[]) => mockTraces(...args),
   useTraceDetail: (...args: unknown[]) => mockTraceDetail(...args),
   usePromoteTraceEval: () => ({
     mutate: vi.fn(),
@@ -125,12 +122,15 @@ vi.mock("./useObservability.js", () => ({
 
 import { AgentNativeI18nProvider } from "@agent-native/core/client/i18n";
 
+import { createToolkitI18nCatalog } from "../i18n.js";
 import {
   ObservabilityDashboard,
   resolveReviewArtifactHref,
   resolveReviewArtifactOpenHref,
 } from "./ObservabilityDashboard.js";
 
+const toolkitI18nCatalog = createToolkitI18nCatalog({ messages: {} });
+
 describe("human review artifact links", () => {
   it("keeps artifact links on the matching first-party app and environment", () => {
     expect(
@@ -442,50 +442,84 @@ describe("ObservabilityDashboard human review", () => {
     vi.unstubAllGlobals();
   });
 
-  it("opens span details, the full conversation, and tab documentation", async () => {
-    mockTraces.mockReturnValue({
-      isLoading: false,
-      data: [
+  it("explains a prompt in plain language and still exposes its raw spans and conversation", async () => {
+    const run = {
+      runId: "run-1",
+      threadId: "thread-1",
+      createdAt: Date.now(),
+      ownerEmail: "owner@example.com",
+      label: "chat",
+      model: "test-model",
+      prompt: "Find the latest releases",
+      status: "success",
+      tokens: {
+        inputTokens: 2,
+        outputTokens: 3,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+      },
+      cost: {
+        cacheReadCents: 0,
+        cacheWriteCents: 0,
+        uncachedInputCents: 1,
+        outputCents: 1,
+        totalCents: 2,
+        estimatedCents: 2,
+        noCacheCents: 2,
+      },
+      modelCalls: 1,
+      tools: [
+        { name: "search", calls: 1, failed: 0, error: null },
         {
-          runId: "run-1",
-          threadId: "thread-1",
-          totalSpans: 2,
-          llmCalls: 1,
-          toolCalls: 1,
-          successfulTools: 1,
-          failedTools: 0,
-          totalDurationMs: 100,
-          totalCostCentsX100: 0,
-          totalInputTokens: 2,
-          totalOutputTokens: 3,
-          model: "test-model",
-          createdAt: Date.now(),
+          name: "broken-search",
+          calls: 1,
+          failed: 1,
+          error: "Search provider returned 503",
         },
       ],
-    });
+      restarts: {
+        count: 0,
+        cents: 0,
+        byCause: {
+          "tool-lookup": { count: 0, cents: 0 },
+          "prefix-changed": { count: 0, cents: 0 },
+        },
+      },
+      parallel: { calls: 0, savedMs: 0 },
+      recoveredErrors: 1,
+      durationMs: 100,
+      feedback: null,
+    };
+    mockUseActionQuery.mockImplementation((actionName) =>
+      actionName === "get-usage-insights"
+        ? {
+            data: {
+              sinceDays: 7,
+              current: { runs: 1, tokens: run.tokens, cost: run.cost },
+              previous: { runs: 0, tokens: run.tokens, cost: run.cost },
+              runs: [run],
+            },
+          }
+        : actionName === "get-usage-run"
+          ? {
+              isLoading: false,
+              data: {
+                ...run,
+                reply: "Here are the three latest releases.",
+                turns: [],
+                scores: [],
+              },
+            }
+          : { data: undefined, isError: false, isSuccess: false },
+    );
     mockTraceDetail.mockReturnValue({
       isLoading: false,
       data: {
-        summary: {
-          runId: "run-1",
-          threadId: "thread-1",
-          totalSpans: 2,
-          llmCalls: 1,
-          toolCalls: 1,
-          successfulTools: 1,
-          failedTools: 0,
-          totalDurationMs: 100,
-          totalCostCentsX100: 0,
-          totalInputTokens: 2,
-          totalOutputTokens: 3,
-          model: "test-model",
-          createdAt: Date.now(),
-        },
+        summary: { runId: "run-1", threadId: "thread-1" },
         spans: [
           {
             id: "span-success",
             runId: "run-1",
-            orgId: "org-a",
             threadId: "thread-1",
             parentSpanId: null,
             spanType: "tool_call",
@@ -529,13 +563,21 @@ describe("ObservabilityDashboard human review", () => {
     await act(async () => {
       root.render(
         <QueryClientProvider client={queryClient}>
-          <AgentNativeI18nProvider persistPreference={false}>
+          <AgentNativeI18nProvider
+            catalog={toolkitI18nCatalog}
+            persistPreference={false}
+          >
             <ObservabilityDashboard />
           </AgentNativeI18nProvider>
         </QueryClientProvider>,
       );
     });
 
+    const buttonWithText = (text: string) =>
+      Array.from(document.querySelectorAll<HTMLButtonElement>("button")).find(
+        (button) => button.textContent?.includes(text),
+      );
+
     const experimentsTab = Array.from(
       container.querySelectorAll<HTMLButtonElement>("button"),
     ).find((button) => button.textContent?.includes("Experiments"));
@@ -552,33 +594,150 @@ describe("ObservabilityDashboard human review", () => {
       container.querySelector<HTMLAnchorElement>('a[href*="#conversations"]'),
     ).toBeTruthy();
 
-    const runRow = Array.from(container.querySelectorAll("tr")).find((row) =>
-      row.textContent?.includes("run-1"),
+    await act(async () => buttonWithText("Find the latest releases")?.click());
+    expect(document.body.textContent).toContain(
+      "Here are the three latest releases.",
+    );
+    expect(document.body.textContent).toContain(
+      "but the agent kept going and finished",
     );
-    await act(async () => (runRow as HTMLTableRowElement | undefined)?.click());
 
+    await act(async () => buttonWithText("Raw trace")?.click());
     const detailsButton = (spanName: string) =>
       Array.from(
-        container.querySelectorAll<HTMLButtonElement>(
+        document.querySelectorAll<HTMLButtonElement>(
           'button[aria-label="View details"]',
         ),
       ).find((button) => button.closest("tr")?.textContent?.includes(spanName));
 
     await act(async () => detailsButton("search")?.click());
-    expect(container.textContent).toContain('"latest releases"');
-    expect(container.textContent).toContain("Found 3 results");
+    expect(document.body.textContent).toContain('"latest releases"');
+    expect(document.body.textContent).toContain("Found 3 results");
 
     await act(async () => detailsButton("broken-search")?.click());
-    expect(container.textContent).toContain("Search provider returned 503");
+    expect(document.body.textContent).toContain("Search provider returned 503");
 
-    await act(async () =>
-      Array.from(container.querySelectorAll<HTMLButtonElement>("button"))
-        .find((button) =>
-          button.textContent?.includes("Open full conversation"),
-        )
-        ?.click(),
-    );
+    await act(async () => buttonWithText("Open full conversation")?.click());
     expect(mockOpenThread).toHaveBeenCalledWith({ threadId: "thread-1" });
+    expect(buttonWithText("Open full conversation")).toBeUndefined();
+  });
+
+  it("reports a failed insights load with a retry instead of loading forever", async () => {
+    const refetch = vi.fn();
+    mockUseActionQuery.mockImplementation((actionName) =>
+      actionName === "get-usage-insights"
+        ? {
+            data: undefined,
+            isError: true,
+            isSuccess: false,
+            isFetching: false,
+            refetch,
+          }
+        : { data: undefined, isError: false, isSuccess: false },
+    );
+
+    await act(async () => {
+      root.render(
+        <QueryClientProvider client={queryClient}>
+          <AgentNativeI18nProvider
+            catalog={toolkitI18nCatalog}
+            persistPreference={false}
+          >
+            <ObservabilityDashboard />
+          </AgentNativeI18nProvider>
+        </QueryClientProvider>,
+      );
+    });
+    expect(container.querySelector('[role="alert"]')?.textContent).toBe(
+      "Couldn't load this. Please try again.",
+    );
+
+    const buttonWithText = (text: string) =>
+      Array.from(container.querySelectorAll("button")).find((button) =>
+        button.textContent?.includes(text),
+      );
+    await act(async () => buttonWithText("Retry")?.click());
+    expect(refetch).toHaveBeenCalledTimes(1);
+
+    await act(async () => activateTab(buttonWithText("Conversations")));
+    expect(container.querySelector('[role="alert"]')?.textContent).toBe(
+      "Couldn't load this. Please try again.",
+    );
+    expect(buttonWithText("Retry")).toBeTruthy();
+  });
+
+  it("says the overview stats cover only the latest prompts when the period has more", async () => {
+    const tokens = {
+      inputTokens: 2,
+      outputTokens: 3,
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+    };
+    const cost = {
+      cacheReadCents: 0,
+      cacheWriteCents: 0,
+      uncachedInputCents: 1,
+      outputCents: 1,
+      totalCents: 2,
+      estimatedCents: 2,
+      noCacheCents: 2,
+    };
+    mockUseActionQuery.mockImplementation((actionName) =>
+      actionName === "get-usage-insights"
+        ? {
+            data: {
+              sinceDays: 7,
+              current: { runs: 30, tokens, cost },
+              previous: { runs: 0, tokens, cost },
+              runs: [
+                {
+                  runId: "run-1",
+                  threadId: "thread-1",
+                  createdAt: Date.now(),
+                  ownerEmail: "owner@example.com",
+                  label: "chat",
+                  model: "test-model",
+                  prompt: "Find the latest releases",
+                  status: "success",
+                  tokens,
+                  cost,
+                  modelCalls: 1,
+                  tools: [],
+                  restarts: {
+                    count: 0,
+                    cents: 0,
+                    byCause: {
+                      "tool-lookup": { count: 0, cents: 0 },
+                      "prefix-changed": { count: 0, cents: 0 },
+                    },
+                  },
+                  parallel: { calls: 0, savedMs: 0 },
+                  recoveredErrors: 0,
+                  durationMs: 100,
+                  feedback: null,
+                },
+              ],
+            },
+          }
+        : { data: undefined, isError: false, isSuccess: false },
+    );
+
+    await act(async () => {
+      root.render(
+        <QueryClientProvider client={queryClient}>
+          <AgentNativeI18nProvider
+            catalog={toolkitI18nCatalog}
+            persistPreference={false}
+          >
+            <ObservabilityDashboard />
+          </AgentNativeI18nProvider>
+        </QueryClientProvider>,
+      );
+    });
+
+    expect(container.textContent).toContain(
+      "Completion, typical time, what Agent-Native handled and the findings use the latest 1 of 30 prompts.",
+    );
   });
 
   it("hides organization human review outside the admin settings surface", async () => {

package/src/app/observability/ObservabilityDashboard.spec.tsx

@@ -24,6 +24,46 @@ const TRACE = {
   createdAt: Date.now(),
 };
 
+const RUN = {
+  runId: TRACE.runId,
+  threadId: TRACE.threadId,
+  createdAt: TRACE.createdAt,
+  ownerEmail: "owner@example.com",
+  label: "chat",
+  model: TRACE.model,
+  prompt: "Say hello",
+  status: "success",
+  tokens: {
+    inputTokens: 8,
+    outputTokens: 4,
+    cacheReadTokens: 0,
+    cacheWriteTokens: 0,
+  },
+  cost: {
+    cacheReadCents: 0,
+    cacheWriteCents: 0,
+    uncachedInputCents: 0.05,
+    outputCents: 0.05,
+    totalCents: 0.1,
+    estimatedCents: 0.1,
+    noCacheCents: 0.1,
+  },
+  modelCalls: 1,
+  tools: [{ name: "search-docs", calls: 1, failed: 0, error: null }],
+  restarts: {
+    count: 0,
+    cents: 0,
+    byCause: {
+      "tool-lookup": { count: 0, cents: 0 },
+      "prefix-changed": { count: 0, cents: 0 },
+    },
+  },
+  parallel: { calls: 0, savedMs: 0 },
+  recoveredErrors: 0,
+  durationMs: TRACE.totalDurationMs,
+  feedback: null,
+};
+
 function jsonResponse(body: unknown, status = 200): Response {
   return new Response(JSON.stringify(body), {
     status,
@@ -42,6 +82,21 @@ beforeEach(() => {
     (input: RequestInfo | URL, init?: RequestInit) => {
       const url = String(input);
       const method = (init?.method ?? "GET").toUpperCase();
+      if (url.includes("/_agent-native/actions/get-usage-insights")) {
+        return Promise.resolve(
+          jsonResponse({
+            sinceDays: 7,
+            current: { runs: 1, tokens: RUN.tokens, cost: RUN.cost },
+            previous: { runs: 0, tokens: RUN.tokens, cost: RUN.cost },
+            runs: [RUN],
+          }),
+        );
+      }
+      if (url.includes("/_agent-native/actions/get-usage-run")) {
+        return Promise.resolve(
+          jsonResponse({ ...RUN, reply: "hello", turns: [], scores: [] }),
+        );
+      }
       if (url.includes("/traces/run-promote-1/promote") && method === "POST") {
         return Promise.resolve(
           jsonResponse({
@@ -119,6 +174,12 @@ afterEach(() => {
   queryClient.clear();
 });
 
+function bodyButton(text: string) {
+  return Array.from(document.body.querySelectorAll("button")).find((button) =>
+    button.textContent?.startsWith(text),
+  );
+}
+
 function renderDashboard() {
   act(() => {
     root.render(
@@ -154,38 +215,37 @@ describe("ObservabilityDashboard promote control", () => {
     );
 
     await vi.waitFor(() => {
-      expect(container.textContent).toContain("run-prom");
+      expect(container.textContent).toContain("Say hello");
     });
 
-    const row = container.querySelector("tbody tr") as HTMLTableRowElement;
-    expect(row).toBeTruthy();
-    act(() => row.click());
+    const promptRow = Array.from(container.querySelectorAll("button")).find(
+      (button) => button.textContent?.startsWith("Say hello"),
+    );
+    act(() => promptRow!.click());
+
+    await vi.waitFor(() => {
+      expect(bodyButton("Raw trace")).toBeTruthy();
+    });
+    act(() => bodyButton("Raw trace")!.click());
 
     await vi.waitFor(() => {
-      const promote = Array.from(container.querySelectorAll("button")).find(
-        (button) => button.textContent === "Promote to eval",
-      );
+      const promote = bodyButton("Promote to eval");
       expect(promote).toBeTruthy();
       expect(promote?.disabled).toBe(false);
     });
 
-    const promote = Array.from(container.querySelectorAll("button")).find(
-      (button) => button.textContent === "Promote to eval",
+    const promote = bodyButton("Promote to eval");
+    const mustContain = document.body.querySelector<HTMLInputElement>(
+      'input[aria-label="Text to check for in the promoted eval reply"]',
+    );
+    expect(mustContain?.placeholder).toBe(
+      "Optional text to check for in the reply…",
     );
-    expect(promote).toBeTruthy();
-    expect(
-      container.querySelector<HTMLInputElement>("input")?.placeholder,
-    ).toBe("Optional text to check for in the reply…");
-    expect(
-      container
-        .querySelector<HTMLInputElement>("input")
-        ?.getAttribute("aria-label"),
-    ).toBe("Text to check for in the promoted eval reply");
     act(() => promote!.click());
 
     await vi.waitFor(() => {
-      expect(container.textContent).toContain("Eval dataset ds-99");
-      expect(container.textContent).toContain(
+      expect(document.body.textContent).toContain("Eval dataset ds-99");
+      expect(document.body.textContent).toContain(
         "agent-native eval promote run-promote-1 --write evals/from-trace.eval.ts",
       );
     });