@agent-native/toolkit 0.205.0 → 0.206.0-nightly-20261009011246

package/src/app/settings/model/ProviderDialog.tsx

@@ -8,8 +8,17 @@ import {
   getAgentProviderOption,
   type AgentProviderId,
 } from "@agent-native/core/client/agent-provider-catalog";
+import { injectedAgentNativeAppId } from "@agent-native/core/client/app-config";
 import { callAction, useActionQuery } from "@agent-native/core/client/hooks";
 import { useFormatters, useT } from "@agent-native/core/client/i18n";
+import {
+  requestCustomKeyOnboardingAbandonment,
+  setCustomKeyOnboardingSetupKind,
+  trackOnboardingEvent,
+  trackCustomKeyOnboardingOutcome,
+  withCustomKeyOnboardingCredentialSave,
+  withCustomKeyOnboardingLocalEndpointSave,
+} from "@agent-native/core/client/onboarding/use-onboarding";
 import { useOrg } from "@agent-native/core/client/org";
 import { Alert, AlertDescription } from "@agent-native/toolkit/ui/alert";
 import { Button } from "@agent-native/toolkit/ui/button";
@@ -40,9 +49,17 @@ import {
   IconServer,
 } from "@tabler/icons-react";
 import { useQueryClient } from "@tanstack/react-query";
-import { useEffect, useId, useMemo, useRef, useState } from "react";
+import {
+  useCallback,
+  useEffect,
+  useId,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 
 import { BrandLogo } from "../infra/logos.js";
+import { currentTemplateId } from "../shell/app-identity.js";
 import { WhoField } from "../WhoField.js";
 import {
   addDialogChoices,
@@ -82,10 +99,73 @@ const PROVIDER_LOGO_IDS: Record<AgentProviderId, string | null> = {
  */
 export type ProviderDialogMode = "add" | "manage" | "add-from-service";
 
+type ProviderSetupTrackingFlow = "chat_setup" | "settings";
+type ProviderSetupEventName =
+  | "integration_key_entry_started"
+  | "integration_key_validation_outcome"
+  | "integration_key_save_outcome";
+type ProviderSetupOutcome =
+  | "started"
+  | "accepted"
+  | "rejected"
+  | "missing_key"
+  | "invalid_endpoint"
+  | "unreachable"
+  | "provider_error"
+  | "error"
+  | "saved"
+  | "failed";
+
+function setupTelemetryAppName(): string {
+  const appId = injectedAgentNativeAppId() ?? currentTemplateId();
+  return appId && /^[a-z0-9][a-z0-9-]{0,63}$/.test(appId)
+    ? appId.replace(/^agent-native-/, "") || "framework"
+    : "framework";
+}
+
+function trackProviderSetupEvent(
+  flow: ProviderSetupTrackingFlow,
+  eventName: ProviderSetupEventName,
+  action: "enter" | "validate" | "save",
+  outcome: ProviderSetupOutcome,
+): void {
+  trackOnboardingEvent(eventName, {
+    flow,
+    app_name: setupTelemetryAppName(),
+    step_id: "connect_ai",
+    method_id: "custom_keys",
+    action,
+    outcome,
+  });
+}
+
+function providerCheckOutcome(
+  result: ProviderModelsCheck,
+): ProviderSetupOutcome {
+  if (result.ok) return "accepted";
+  switch (result.code) {
+    case "rejected":
+    case "wrong-provider":
+      return "rejected";
+    case "missing-key":
+      return "missing_key";
+    case "invalid-endpoint":
+      return "invalid_endpoint";
+    case "unreachable":
+      return "unreachable";
+    case "provider-error":
+      return "provider_error";
+    default:
+      return "error";
+  }
+}
+
 export interface ProviderDialogProps {
   open: boolean;
   onOpenChange: (open: boolean) => void;
   mode: ProviderDialogMode;
+  /** Preserve the chat setup handoff through the first provider dialog. */
+  trackingFlow?: ProviderSetupTrackingFlow;
   /** Required for `manage` and `add-from-service`; the first choice for `add`. */
   provider?: AgentProviderId;
   /** `manage`: which saved key, the personal (`user`) or organization (`org`) one. */
@@ -134,14 +214,58 @@ function unique(models: readonly string[]): string[] {
  * the checked models in one step.
  */
 export function ProviderDialog(props: ProviderDialogProps) {
+  const savePending = useRef(false);
+  const selectedProvider = useRef<AgentProviderId>(
+    props.provider ?? props.providers?.[0] ?? "anthropic",
+  );
+  const reportSelectedProvider = useCallback((provider: AgentProviderId) => {
+    selectedProvider.current = provider;
+    setCustomKeyOnboardingSetupKind(
+      provider === "ollama" ? "local_endpoint" : "credential",
+    );
+  }, []);
+  const dismiss = () => {
+    if (selectedProvider.current === "ollama" && !savePending.current) {
+      trackCustomKeyOnboardingOutcome("local_endpoint_skipped");
+    } else if (savePending.current) {
+      requestCustomKeyOnboardingAbandonment();
+    } else {
+      trackCustomKeyOnboardingOutcome("credential_skipped");
+    }
+    props.onOpenChange(false);
+  };
+
   return (
-    <Dialog open={props.open} onOpenChange={props.onOpenChange}>
-      {props.open ? <ProviderDialogContent {...props} /> : null}
+    <Dialog
+      open={props.open}
+      onOpenChange={(open) => {
+        if (open) props.onOpenChange(true);
+        else dismiss();
+      }}
+    >
+      {props.open ? (
+        <ProviderDialogContent
+          {...props}
+          onDismiss={dismiss}
+          onProviderChange={reportSelectedProvider}
+          onSavingChange={(saving) => {
+            savePending.current = saving;
+          }}
+        />
+      ) : null}
     </Dialog>
   );
 }
 
-function ProviderDialogContent(props: ProviderDialogProps) {
+interface ProviderDialogInternalProps {
+  onDismiss: () => void;
+  onProviderChange: (provider: AgentProviderId) => void;
+  onSavingChange: (saving: boolean) => void;
+}
+
+function ProviderDialogContent(
+  props: ProviderDialogProps & ProviderDialogInternalProps,
+) {
   const t = useT();
   const listing = useActionQuery<ModelProvidersListing>(
     "list-model-providers" as never,
@@ -209,7 +333,7 @@ function ProviderDialogContent(props: ProviderDialogProps) {
   );
 }
 
-interface FormProps extends ProviderDialogProps {
+interface FormProps extends ProviderDialogProps, ProviderDialogInternalProps {
   title: string;
   listing: ModelProvidersListing;
   models: ProviderModelsRead;
@@ -217,11 +341,15 @@ interface FormProps extends ProviderDialogProps {
 
 function ProviderDialogForm({
   mode,
+  trackingFlow = "settings",
   provider: requestedProvider,
   scope: requestedScope,
   serviceLabel,
   providers: providerChoices,
   onOpenChange,
+  onDismiss,
+  onProviderChange,
+  onSavingChange,
   onSaved,
   onRemoved,
   title,
@@ -286,6 +414,7 @@ function ProviderDialogForm({
   );
   const [keyValue, setKeyValue] = useState("");
   const [keyError, setKeyError] = useState(false);
+  const keyEntryTrackedRef = useRef(false);
   const savedEndpoint = isOpenAi ? (existing?.endpoint ?? "") : "";
   const [endpointOpen, setEndpointOpen] = useState(!!savedEndpoint);
   const [endpoint, setEndpoint] = useState(savedEndpoint);
@@ -313,9 +442,22 @@ function ProviderDialogForm({
   const requestRef = useRef(0);
   const fromService = mode === "add-from-service";
 
+  useEffect(() => {
+    onProviderChange(provider);
+  }, [onProviderChange, provider]);
+
   // A pasted key (or Ollama endpoint) is checked as it's entered, by asking
   // the provider which models it reaches.
   useEffect(() => {
+    if (keyValue.trim() && !keyEntryTrackedRef.current) {
+      keyEntryTrackedRef.current = true;
+      trackProviderSetupEvent(
+        trackingFlow,
+        "integration_key_entry_started",
+        "enter",
+        "started",
+      );
+    }
     if (!replacing) return;
     const value = keyValue.trim();
     const request = ++requestRef.current;
@@ -336,7 +478,16 @@ function ProviderDialogForm({
         .then((result) => {
           if (request !== requestRef.current) return;
           setCheck(toCheckState(result));
+          trackProviderSetupEvent(
+            trackingFlow,
+            "integration_key_validation_outcome",
+            "validate",
+            providerCheckOutcome(result),
+          );
           if (!result.ok) return;
+          if (provider !== "ollama") {
+            trackCustomKeyOnboardingOutcome("credential_validated");
+          }
           setChecked((previous) => {
             const kept = previous.filter((model) =>
               result.models.includes(model),
@@ -351,6 +502,12 @@ function ProviderDialogForm({
             state: "error",
             message: err instanceof Error ? err.message : String(err),
           });
+          trackProviderSetupEvent(
+            trackingFlow,
+            "integration_key_validation_outcome",
+            "validate",
+            "error",
+          );
         });
     }, CHECK_DEBOUNCE_MS);
     return () => window.clearTimeout(timer);
@@ -362,6 +519,7 @@ function ProviderDialogForm({
     replacing,
     models,
     fromService,
+    trackingFlow,
   ]);
 
   // Models checked when the dialog opened stay listed after a check, so one
@@ -392,6 +550,7 @@ function ProviderDialogForm({
       addChoices.find((choice) => choice.provider === next)?.replaces ?? null;
     const nextEndpoint = next === "openai" ? (target?.endpoint ?? "") : "";
     setProvider(next);
+    onProviderChange(next);
     setKeyValue((value) => (keepValue ? value : ""));
     setKeyError(false);
     setChecked(
@@ -411,10 +570,22 @@ function ProviderDialogForm({
         if (result.ok) {
           setCheck(toCheckState(result));
           setRecheck({ checkedAt: result.checkedAt });
+          trackProviderSetupEvent(
+            trackingFlow,
+            "integration_key_validation_outcome",
+            "validate",
+            providerCheckOutcome(result),
+          );
           return;
         }
         setRecheck("idle");
         setCheck(toCheckState(result));
+        trackProviderSetupEvent(
+          trackingFlow,
+          "integration_key_validation_outcome",
+          "validate",
+          providerCheckOutcome(result),
+        );
         if (result.code === "rejected" || result.code === "wrong-provider") {
           setSavedRejected(true);
           setReplacing(true);
@@ -425,6 +596,12 @@ function ProviderDialogForm({
       .catch((err: unknown) => {
         setRecheck("idle");
         setError(err instanceof Error ? err.message : String(err));
+        trackProviderSetupEvent(
+          trackingFlow,
+          "integration_key_validation_outcome",
+          "validate",
+          "error",
+        );
       });
   };
 
@@ -447,26 +624,39 @@ function ProviderDialogForm({
       mode !== "manage" || !sameModels(checked, initialSelection ?? []);
 
     setSaving(true);
-    let keySaved = false;
+    onSavingChange(true);
+    let settingsSaved = false;
     try {
       if (replacing) {
-        await saveAgentEngineProviderSettings({
-          provider,
-          ...(isOllama ? { baseUrl: value } : { apiKey: value }),
-          ...(gateway ? { baseUrl: gateway } : {}),
-          ...(isOpenAi && !gateway && existing?.endpoint
-            ? { clearBaseUrl: true }
-            : {}),
-          scope,
-        });
-        keySaved = true;
+        const saveProviderSettings = () =>
+          saveAgentEngineProviderSettings({
+            provider,
+            ...(isOllama ? { baseUrl: value } : { apiKey: value }),
+            ...(gateway ? { baseUrl: gateway } : {}),
+            ...(isOpenAi && !gateway && existing?.endpoint
+              ? { clearBaseUrl: true }
+              : {}),
+            scope,
+          });
+        if (isOllama) {
+          await withCustomKeyOnboardingLocalEndpointSave(saveProviderSettings);
+        } else {
+          await withCustomKeyOnboardingCredentialSave(saveProviderSettings);
+        }
+        settingsSaved = true;
+        trackProviderSetupEvent(
+          trackingFlow,
+          "integration_key_save_outcome",
+          "save",
+          "saved",
+        );
       } else if (endpointChanged) {
         await saveAgentEngineProviderSettings({
           provider,
           ...(gateway ? { baseUrl: gateway } : { clearBaseUrl: true }),
           scope,
         });
-        keySaved = true;
+        settingsSaved = true;
       }
       if (modelsChanged) {
         await callAction(
@@ -484,12 +674,23 @@ function ProviderDialogForm({
       onOpenChange(false);
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      setError(keySaved ? t(`${K}modelsSaveFailed`, { message }) : message);
-      if (keySaved) {
+      if (replacing && !settingsSaved) {
+        trackProviderSetupEvent(
+          trackingFlow,
+          "integration_key_save_outcome",
+          "save",
+          "failed",
+        );
+      }
+      setError(
+        settingsSaved ? t(`${K}modelsSaveFailed`, { message }) : message,
+      );
+      if (settingsSaved) {
         void queryClient.invalidateQueries({ queryKey: ["action"] });
       }
     } finally {
       setSaving(false);
+      onSavingChange(false);
     }
   };
 
@@ -508,11 +709,7 @@ function ProviderDialogForm({
           {t(`${K}restricted`)}
         </p>
         <DialogFooter>
-          <Button
-            type="button"
-            variant="secondary"
-            onClick={() => onOpenChange(false)}
-          >
+          <Button type="button" variant="secondary" onClick={onDismiss}>
             {t(`${K}cancel`)}
           </Button>
         </DialogFooter>
@@ -607,6 +804,9 @@ function ProviderDialogForm({
                 onChange={(event) => {
                   setKeyValue(event.target.value);
                   setKeyError(false);
+                  if (event.target.value.trim() && !isOllama) {
+                    trackCustomKeyOnboardingOutcome("credential_entry_started");
+                  }
                 }}
               />
             ) : (
@@ -762,11 +962,7 @@ function ProviderDialogForm({
               {t(`${K}removeProvider`)}
             </Button>
           ) : null}
-          <Button
-            type="button"
-            variant="secondary"
-            onClick={() => onOpenChange(false)}
-          >
+          <Button type="button" variant="secondary" onClick={onDismiss}>
             {t(`${K}cancel`)}
           </Button>
           <Button type="submit" disabled={saving || !ready}>

package/src/app/settings/useBuilderStatus.spec.tsx

@@ -12,6 +12,7 @@ import {
   isBuilderConnectComplete,
   useBuilderStatus,
   useBuilderConnectFlow,
+  requestBuilderAccountActivation,
   withBuilderConnectTrackingParams,
   type BuilderConnectionScope,
 } from "./useBuilderStatus.js";
@@ -292,6 +293,7 @@ describe("useBuilderStatus", () => {
   afterEach(() => {
     act(() => root.unmount());
     container.remove();
+    Reflect.deleteProperty(window.location, "hostname");
     vi.unstubAllGlobals();
   });
 
@@ -498,6 +500,7 @@ describe("useBuilderConnectFlow", () => {
   afterEach(() => {
     act(() => root.unmount());
     container.remove();
+    Reflect.deleteProperty(window.location, "hostname");
     vi.useRealTimers();
     vi.unstubAllGlobals();
   });
@@ -776,6 +779,12 @@ describe("useBuilderConnectFlow", () => {
 
     for (const host of ["desktop", "embedded", "browser"] as const) {
       it(`creates the account with one request and no popup (${host})`, async () => {
+        Object.defineProperty(window.location, "hostname", {
+          configurable: true,
+          value: "design.custom-domain.com",
+        });
+        vi.stubGlobal("__AGENT_NATIVE_APP_ID__", "agent-native-clips");
+        vi.stubGlobal("__AGENT_NATIVE_TEMPLATE__", "clips");
         setUserAgent(
           host === "desktop"
             ? "Mozilla/5.0 AgentNativeDesktop/1.0"
@@ -809,11 +818,48 @@ describe("useBuilderConnectFlow", () => {
         expect(posts[0]!.url.searchParams.get("agentNativeFlow")).toBe(
           "connect_llm",
         );
+        expect(posts[0]!.url.searchParams.get("agentNativeApp")).toBe(
+          "agent-native-clips",
+        );
+        expect(posts[0]!.url.searchParams.get("agentNativeTemplate")).toBe(
+          "clips",
+        );
         expect(container.textContent).toContain("configured idle resolved");
         expect(onConnected).toHaveBeenCalledOnce();
       });
     }
 
+    it.each([
+      "localhost",
+      "127.0.0.1",
+      "www.agent-native.com",
+      "design.custom-domain.com",
+    ])("does not infer signup attribution from %s", async (hostname) => {
+      Object.defineProperty(window.location, "hostname", {
+        configurable: true,
+        value: hostname,
+      });
+      const posts = mockActivation(() => activationResponse(200, { ok: true }));
+
+      await requestBuilderAccountActivation({ provisioningToken });
+
+      expect(posts).toHaveLength(1);
+      expect(posts[0]!.url.searchParams.get("agentNativeApp")).toBeNull();
+      expect(posts[0]!.url.searchParams.get("agentNativeTemplate")).toBeNull();
+    });
+
+    it("does not infer a template from the injected app ID", async () => {
+      vi.stubGlobal("__AGENT_NATIVE_APP_ID__", "private-workspace");
+      const posts = mockActivation(() => activationResponse(200, { ok: true }));
+
+      await requestBuilderAccountActivation({ provisioningToken });
+
+      expect(posts[0]!.url.searchParams.get("agentNativeApp")).toBe(
+        "private-workspace",
+      );
+      expect(posts[0]!.url.searchParams.get("agentNativeTemplate")).toBeNull();
+    });
+
     it("activates an account for the organization's connection", async () => {
       const posts = mockActivation(() =>
         activationResponse(200, { ok: true, scope: "org" }),
@@ -2604,6 +2650,9 @@ describe("useBuilderConnectFlow", () => {
 
     expect(container.textContent).toContain("not-configured idle");
     expect(container.textContent).toContain("Didn't hear back from Builder");
+    expect(
+      container.querySelector('[data-testid="error-kind"]')?.textContent,
+    ).toBe("connection");
   });
 
   it("clears a stale status-read error after a readable incomplete callback status", async () => {

package/src/app/settings/useBuilderStatus.ts

@@ -1,5 +1,6 @@
 import { trackEvent } from "@agent-native/core/client/analytics";
 import { agentNativePath } from "@agent-native/core/client/api-path";
+import { injectedAgentNativeAppId } from "@agent-native/core/client/app-config";
 import { getCallbackOrigin } from "@agent-native/core/client/frame";
 import { scheduleAfterPaint } from "@agent-native/core/client/hooks";
 import { usePollLoop } from "@agent-native/core/client/hooks";
@@ -297,6 +298,7 @@ const BUILDER_AGENT_NATIVE_FLOW_PARAM = "agentNativeFlow";
 const BUILDER_AGENT_NATIVE_CONNECT_SOURCE_PARAM = "agentNativeConnectSource";
 const BUILDER_AGENT_NATIVE_APP_PARAM = "agentNativeApp";
 const BUILDER_AGENT_NATIVE_TEMPLATE_PARAM = "agentNativeTemplate";
+declare const __AGENT_NATIVE_TEMPLATE__: string | undefined;
 const BUILDER_SIGNUP_SOURCE = "agent-native";
 const STATUS_CONNECT_URL_TTL_MS = 9 * 60 * 1000;
 
@@ -354,18 +356,16 @@ function inferBuilderConnectTrackingIdentity(options: {
   const app =
     normalizeTrackingSlug(options.app) ??
     normalizeTrackingSlug(env.VITE_AGENT_NATIVE_APP) ??
-    (typeof window !== "undefined"
-      ? normalizeTrackingSlug(window.location.hostname.split(".")[0])
-      : null);
+    normalizeTrackingSlug(injectedAgentNativeAppId());
   const template =
     normalizeTrackingSlug(options.template) ??
     normalizeTrackingSlug(env.VITE_AGENT_NATIVE_TEMPLATE) ??
     normalizeTrackingSlug(env.VITE_APP_TEMPLATE) ??
-    (app?.startsWith("agent-native-")
-      ? normalizeTrackingSlug(app.slice("agent-native-".length))
-      : app && app !== "localhost"
-        ? app
-        : null);
+    normalizeTrackingSlug(
+      typeof __AGENT_NATIVE_TEMPLATE__ === "string"
+        ? __AGENT_NATIVE_TEMPLATE__
+        : undefined,
+    );
 
   return { app, template };
 }
@@ -722,9 +722,15 @@ export async function requestBuilderAccountActivation({
   signal?: AbortSignal;
 }): Promise<BuilderAccountActivationResult> {
   const origin = getCallbackOrigin() || window.location.origin;
+  const { app, template } = inferBuilderConnectTrackingIdentity({});
   const url = withBuilderConnectTrackingParams(
     new URL(agentNativePath("/_agent-native/builder/provision"), origin).href,
-    { source, flow },
+    {
+      source,
+      flow,
+      app: app ?? undefined,
+      template: template ?? undefined,
+    },
   );
   let response: Response;
   try {
@@ -1812,7 +1818,7 @@ export function useBuilderConnectFlow(
         if (s) {
           setError(
             "Didn't hear back from Builder in 5 minutes. Allow popups and try again.",
-            "launch",
+            "connection",
           );
         }
       }