@agent-native/toolkit 0.205.0 → 0.206.0-nightly-20261009011246

package/src/app/settings/model/ModelSettingsPage.tsx

@@ -1,6 +1,6 @@
 import type { ProviderKeyPolicyStatus } from "@agent-native/core/agent/actions/manage-provider-key-policy";
 import { CHATGPT_SUBSCRIPTION_ENGINE_NAME } from "@agent-native/core/agent/chatgpt-subscription-contract";
-import { upgradeModelToLatestSupportedVersion } from "@agent-native/core/agent/model-version";
+import { upgradeModelForProvider } from "@agent-native/core/agent/model-version";
 import {
   setAgentEngineDefaultModel,
   type AgentEngineKeyScope,
@@ -42,7 +42,16 @@ import {
 } from "@agent-native/toolkit/ui/tooltip";
 import { IconCpu, IconLock, IconPlus } from "@tabler/icons-react";
 import { useQuery, useQueryClient } from "@tanstack/react-query";
-import { useEffect, useId, useMemo, useState, type ReactNode } from "react";
+import {
+  useCallback,
+  useEffect,
+  useId,
+  useMemo,
+  useRef,
+  useState,
+  type ReactNode,
+} from "react";
+import { useLocation, useNavigate } from "react-router";
 
 import { ErrorRow, SettingsEmpty } from "../../resources/index.js";
 import { DeferredBuilderConnectPopover } from "../deferred-builder-connect-popover.js";
@@ -90,6 +99,11 @@ type DialogState =
   | { mode: "add" }
   | { mode: "manage"; provider: AgentProviderId; scope: AgentEngineKeyScope };
 
+type ActiveDialogState = {
+  dialog: DialogState;
+  trackingFlow: "chat_setup" | "settings";
+};
+
 type ChatGPTSubscriptionStatusRead = {
   connected: boolean;
   activeAccountId: string | null;
@@ -156,6 +170,12 @@ function useChatGPTModels(
  */
 export default function ModelSettingsPage(_props: SettingsPageProps) {
   const t = useT();
+  const location = useLocation();
+  const navigate = useNavigate();
+  const chatSetupTrackingPending = useRef(
+    (location.state as { providerSetupTrackingFlow?: unknown } | null)
+      ?.providerSetupTrackingFlow === "chat_setup",
+  );
   const org = useOrg();
   const listing = useActionQuery<ModelProvidersListing>(
     "list-model-providers" as never,
@@ -168,8 +188,42 @@ export default function ModelSettingsPage(_props: SettingsPageProps) {
     trackingSource: "settings_model",
     trackingFlow: "connect_llm",
   });
-  const [dialog, setDialog] = useState<DialogState | null>(null);
+  const [dialog, setDialog] = useState<ActiveDialogState | null>(null);
   const [removing, setRemoving] = useState<ProviderKeyRow | null>(null);
+  const openProviderDialog = useCallback(
+    (nextDialog: DialogState) => {
+      const trackingFlow = chatSetupTrackingPending.current
+        ? "chat_setup"
+        : "settings";
+      if (chatSetupTrackingPending.current) {
+        chatSetupTrackingPending.current = false;
+        const locationState =
+          location.state && typeof location.state === "object"
+            ? Object.fromEntries(
+                Object.entries(location.state).filter(
+                  ([key]) => key !== "providerSetupTrackingFlow",
+                ),
+              )
+            : null;
+        navigate(
+          {
+            pathname: location.pathname,
+            search: location.search,
+            hash: location.hash,
+          },
+          {
+            replace: true,
+            state:
+              locationState && Object.keys(locationState).length > 0
+                ? locationState
+                : null,
+          },
+        );
+      }
+      setDialog({ dialog: nextDialog, trackingFlow });
+    },
+    [location.hash, location.pathname, location.search, navigate],
+  );
 
   const chatgptLab = useLabState("chatgpt-subscription");
   const chatgptStatus = useActionQuery<ChatGPTSubscriptionStatusRead>(
@@ -212,10 +266,12 @@ export default function ModelSettingsPage(_props: SettingsPageProps) {
     () => ({
       action:
         canAdd && !needsProvider ? (
-          <AddProviderButton onClick={() => setDialog({ mode: "add" })} />
+          <AddProviderButton
+            onClick={() => openProviderDialog({ mode: "add" })}
+          />
         ) : undefined,
     }),
-    [canAdd, needsProvider],
+    [canAdd, needsProvider, openProviderDialog],
   );
   useSettingsPageHeader(header);
 
@@ -240,7 +296,11 @@ export default function ModelSettingsPage(_props: SettingsPageProps) {
       : { status: "loading" };
   const orgName = org.data?.orgName ?? "";
   const openManage = (row: ProviderKeyRow) =>
-    setDialog({ mode: "manage", provider: row.provider, scope: row.key.scope });
+    openProviderDialog({
+      mode: "manage",
+      provider: row.provider,
+      scope: row.key.scope,
+    });
 
   return (
     <TooltipProvider delayDuration={200}>
@@ -251,7 +311,7 @@ export default function ModelSettingsPage(_props: SettingsPageProps) {
               listing={data}
               canAdd={canAdd}
               builder={builder}
-              onAdd={() => setDialog({ mode: "add" })}
+              onAdd={() => openProviderDialog({ mode: "add" })}
             />
           </SettingsGroup>
         ) : data.hasOrganization ? (
@@ -295,12 +355,16 @@ export default function ModelSettingsPage(_props: SettingsPageProps) {
       </div>
       <ProviderDialog
         open={dialog !== null}
+        trackingFlow={dialog?.trackingFlow ?? "settings"}
         onOpenChange={(open) => {
           if (!open) setDialog(null);
         }}
-        mode={dialog?.mode ?? "add"}
-        {...(dialog?.mode === "manage"
-          ? { provider: dialog.provider, scope: dialog.scope }
+        mode={dialog?.dialog.mode ?? "add"}
+        {...(dialog?.dialog.mode === "manage"
+          ? {
+              provider: dialog.dialog.provider,
+              scope: dialog.dialog.scope,
+            }
           : {})}
       />
       {removing ? (
@@ -802,7 +866,11 @@ function DefaultModelRow({
           engine: listing.defaultModel.engine,
           model:
             (!preserveCustomModels &&
-              upgradeModelToLatestSupportedVersion(model, supportedModels)) ||
+              upgradeModelForProvider(
+                model,
+                supportedModels,
+                listing.defaultModel.engine,
+              )) ||
             model,
         };
       })()

package/src/app/settings/model/ProviderDialog.spec.tsx

@@ -23,6 +23,12 @@ const keyMock = vi.hoisted(() => ({
   deleteAgentEngineProviderSettings: vi.fn(),
 }));
 const callActionMock = vi.hoisted(() => vi.fn());
+const setupTelemetryMock = vi.hoisted(() => vi.fn());
+const onboardingOutcomeMock = vi.hoisted(() => vi.fn());
+const onboardingAbandonmentRequestMock = vi.hoisted(() => vi.fn());
+const onboardingSetupKindMock = vi.hoisted(() => vi.fn());
+const credentialSaveBoundaryMock = vi.hoisted(() => vi.fn());
+const localEndpointSaveBoundaryMock = vi.hoisted(() => vi.fn());
 
 vi.mock("@agent-native/core/client/hooks", () => ({
   useActionQuery: (name: string) => ({
@@ -42,6 +48,29 @@ vi.mock("@agent-native/core/client/hooks", () => ({
 
 vi.mock("@agent-native/core/client/agent-engine-key", () => keyMock);
 
+vi.mock("@agent-native/core/client/onboarding/use-onboarding", () => ({
+  trackOnboardingEvent: setupTelemetryMock,
+  requestCustomKeyOnboardingAbandonment: onboardingAbandonmentRequestMock,
+  setCustomKeyOnboardingSetupKind: onboardingSetupKindMock,
+  trackCustomKeyOnboardingOutcome: onboardingOutcomeMock,
+  withCustomKeyOnboardingCredentialSave: async (
+    save: () => Promise<unknown>,
+  ) => {
+    credentialSaveBoundaryMock();
+    const result = await save();
+    onboardingOutcomeMock("credential_saved");
+    return result;
+  },
+  withCustomKeyOnboardingLocalEndpointSave: async (
+    save: () => Promise<unknown>,
+  ) => {
+    localEndpointSaveBoundaryMock();
+    const result = await save();
+    onboardingOutcomeMock("local_endpoint_saved");
+    return result;
+  },
+}));
+
 vi.mock("@agent-native/core/client/org", () => ({
   useOrg: () => ({ data: { orgName: "Acme" }, isLoading: false }),
 }));
@@ -187,6 +216,12 @@ describe("ProviderDialog", () => {
       .mockReset()
       .mockResolvedValue(undefined);
     callActionMock.mockReset().mockResolvedValue({});
+    setupTelemetryMock.mockReset();
+    onboardingOutcomeMock.mockReset();
+    onboardingAbandonmentRequestMock.mockReset();
+    onboardingSetupKindMock.mockReset();
+    credentialSaveBoundaryMock.mockReset();
+    localEndpointSaveBoundaryMock.mockReset();
     container = document.createElement("div");
     document.body.appendChild(container);
     root = createRoot(container);
@@ -275,6 +310,54 @@ describe("ProviderDialog", () => {
     expect(onOpenChange).toHaveBeenCalledWith(false);
   });
 
+  it("tracks chat setup key entry and save without recording the key", async () => {
+    keyMock.fetchProviderModels.mockResolvedValue({
+      ok: true,
+      provider: "anthropic",
+      models: ["claude-sonnet-5"],
+      checkedAt: 1,
+    });
+    render({ provider: "anthropic", trackingFlow: "chat_setup" });
+
+    typeInto(inputByLabel("API key"), "sk-ant-test-0000");
+    await vi.waitFor(() => {
+      expect(setupTelemetryMock).toHaveBeenCalledWith(
+        "integration_key_entry_started",
+        expect.objectContaining({
+          flow: "chat_setup",
+          app_name: expect.any(String),
+          step_id: "connect_ai",
+          method_id: "custom_keys",
+          action: "enter",
+          outcome: "started",
+        }),
+      );
+      expect(setupTelemetryMock).toHaveBeenCalledWith(
+        "integration_key_validation_outcome",
+        expect.objectContaining({
+          flow: "chat_setup",
+          action: "validate",
+          outcome: "accepted",
+        }),
+      );
+    });
+
+    await act(async () => button("Add provider").click());
+    await vi.waitFor(() => {
+      expect(setupTelemetryMock).toHaveBeenCalledWith(
+        "integration_key_save_outcome",
+        expect.objectContaining({
+          flow: "chat_setup",
+          action: "save",
+          outcome: "saved",
+        }),
+      );
+    });
+    expect(JSON.stringify(setupTelemetryMock.mock.calls)).not.toContain(
+      "sk-ant-test-0000",
+    );
+  });
+
   it("keeps Add disabled until the key checks out and keeps a failed save open", async () => {
     keyMock.fetchProviderModels.mockResolvedValue({
       ok: true,
@@ -306,6 +389,49 @@ describe("ProviderDialog", () => {
     expect(onOpenChange).not.toHaveBeenCalledWith(false);
   });
 
+  it("allows leaving a provider dialog while preserving its pending save outcome", async () => {
+    let resolveSave!: () => void;
+    keyMock.saveAgentEngineProviderSettings.mockReturnValue(
+      new Promise<void>((resolve) => {
+        resolveSave = resolve;
+      }),
+    );
+    keyMock.fetchProviderModels.mockResolvedValue({
+      ok: true,
+      provider: "anthropic",
+      models: ["claude-sonnet-5"],
+      checkedAt: 1,
+    });
+    const { onOpenChange } = render({ provider: "anthropic" });
+
+    typeInto(inputByLabel("API key"), "sk-ant-test-0000");
+    await vi.waitFor(() => {
+      expect(button("Add provider").disabled).toBe(false);
+    });
+    await act(async () => button("Add provider").click());
+    await vi.waitFor(() => {
+      expect(keyMock.saveAgentEngineProviderSettings).toHaveBeenCalled();
+    });
+
+    await act(async () => button("Close").click());
+    expect(onOpenChange).toHaveBeenCalledWith(false);
+    expect(onboardingAbandonmentRequestMock).toHaveBeenCalledTimes(1);
+    expect(
+      onboardingOutcomeMock.mock.calls.map(([outcome]) => outcome),
+    ).not.toContain("credential_skipped");
+
+    await act(async () => {
+      resolveSave();
+      await vi.waitFor(() => expect(onOpenChange).toHaveBeenCalledWith(false));
+    });
+    const outcomes = onboardingOutcomeMock.mock.calls.map(
+      ([outcome]) => outcome,
+    );
+    expect(outcomes).toContain("credential_saved");
+    expect(outcomes).not.toContain("credential_skipped");
+    expect(credentialSaveBoundaryMock).toHaveBeenCalledTimes(1);
+  });
+
   it("locks members to a personal key", async () => {
     state.listing = listing({ canManageOrg: false });
     keyMock.fetchProviderModels.mockResolvedValue({
@@ -387,9 +513,10 @@ describe("ProviderDialog", () => {
       models: ["llama3.1:latest"],
       checkedAt: 1,
     });
-    render({ provider: "ollama" });
+    const { onSaved } = render({ provider: "ollama" });
     expect(document.body.textContent).toContain("No API key required.");
     typeInto(inputByLabel("Endpoint URL"), "http://ollama.internal:11434");
+    expect(onboardingOutcomeMock).not.toHaveBeenCalled();
     await vi.waitFor(() => {
       expect(document.body.textContent).toContain("llama3.1:latest");
     });
@@ -397,6 +524,36 @@ describe("ProviderDialog", () => {
       provider: "ollama",
       baseUrl: "http://ollama.internal:11434",
     });
+    expect(onboardingOutcomeMock).not.toHaveBeenCalled();
+
+    await act(async () => button("Add provider").click());
+    await vi.waitFor(() => expect(onSaved).toHaveBeenCalled());
+
+    expect(keyMock.saveAgentEngineProviderSettings).toHaveBeenCalledWith({
+      provider: "ollama",
+      baseUrl: "http://ollama.internal:11434",
+      scope: "org",
+    });
+    expect(credentialSaveBoundaryMock).not.toHaveBeenCalled();
+    expect(localEndpointSaveBoundaryMock).toHaveBeenCalledOnce();
+    expect(onboardingOutcomeMock).toHaveBeenCalledWith("local_endpoint_saved");
+    expect(onboardingOutcomeMock).not.toHaveBeenCalledWith("credential_saved");
+  });
+
+  it("classifies dismissing Ollama setup separately from skipping credentials", () => {
+    render({ provider: "ollama" });
+
+    expect(onboardingSetupKindMock).toHaveBeenCalledWith("local_endpoint");
+
+    act(() => button("Cancel").click());
+
+    expect(onboardingOutcomeMock).toHaveBeenCalledExactlyOnceWith(
+      "local_endpoint_skipped",
+    );
+    expect(onboardingOutcomeMock).not.toHaveBeenCalledWith(
+      "credential_skipped",
+    );
+    expect(onboardingAbandonmentRequestMock).not.toHaveBeenCalled();
   });
 
   it("manages a saved key: masked, checked on save, and models only", async () => {
@@ -442,6 +599,40 @@ describe("ProviderDialog", () => {
     });
   });
 
+  it("does not count an endpoint-only update as a credential save", async () => {
+    state.listing = listing(
+      {},
+      {
+        openai: {
+          org: {
+            scope: "org",
+            masked: "••••9f3a",
+            updatedAt: 1,
+            endpoint: "https://old.example",
+          },
+        },
+      },
+    );
+    state.models = models({ openai: { org: ["model-a"] } });
+    const { onSaved } = render({
+      mode: "manage",
+      provider: "openai",
+      scope: "org",
+    });
+
+    typeInto(inputByLabel("Endpoint URL"), "https://gateway.example");
+    await act(async () => button("Save").click());
+    await vi.waitFor(() => expect(onSaved).toHaveBeenCalled());
+
+    expect(keyMock.saveAgentEngineProviderSettings).toHaveBeenCalledWith({
+      provider: "openai",
+      baseUrl: "https://gateway.example",
+      scope: "org",
+    });
+    expect(credentialSaveBoundaryMock).not.toHaveBeenCalled();
+    expect(onboardingOutcomeMock).not.toHaveBeenCalledWith("credential_saved");
+  });
+
   it("asks for a new key when the saved one was rejected", () => {
     state.listing = listing(
       {},