@modelnex/sdk 0.5.14 → 0.5.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -2114,7 +2114,7 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
2114
2114
  }
2115
2115
 
2116
2116
  // src/constants.ts
2117
- var DEFAULT_MODELNEX_SERVER_URL = "https://modelnex-server-production.up.railway.app";
2117
+ var DEFAULT_MODELNEX_SERVER_URL = "https://api.modelnex.com";
2118
2118
 
2119
2119
  // src/hooks/useRunCommand.ts
2120
2120
  import { useCallback as useCallback5, useContext as useContext2 } from "react";
@@ -2416,6 +2416,7 @@ function isTourEligible(tour, userProfile) {
2416
2416
 
2417
2417
  // src/hooks/useTourPlayback.ts
2418
2418
  import { useState as useState7, useRef as useRef8, useCallback as useCallback7, useEffect as useEffect11, useContext as useContext4 } from "react";
2419
+ import { io as io2 } from "socket.io-client";
2419
2420
 
2420
2421
  // src/utils/retryLookup.ts
2421
2422
  var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -2944,6 +2945,12 @@ function useTourPlayback({
2944
2945
  const [serverState, setServerState] = useState7(null);
2945
2946
  const ctx = useContext4(ModelNexContext);
2946
2947
  const devMode = ctx?.devMode;
2948
+ const devModeRef = useRef8(devMode);
2949
+ devModeRef.current = devMode;
2950
+ const userProfileRef = useRef8(userProfile);
2951
+ userProfileRef.current = userProfile;
2952
+ const experienceTypeRef = useRef8(experienceType);
2953
+ experienceTypeRef.current = experienceType;
2947
2954
  const tourRef = useRef8(null);
2948
2955
  const stepIndexRef = useRef8(0);
2949
2956
  const skipRequestedRef = useRef8(false);
@@ -2983,408 +2990,391 @@ function useTourPlayback({
2983
2990
  useEffect11(() => {
2984
2991
  if (disabled) return;
2985
2992
  if (typeof window === "undefined") return;
2986
- import("socket.io-client").then((ioModule) => {
2987
- const io2 = ioModule.default || ioModule;
2988
- const socket = io2(serverUrl, {
2989
- path: "/socket.io",
2990
- // standard
2991
- transports: resolveSocketIoTransports(serverUrl, "polling-first")
2992
- });
2993
- socketRef.current = socket;
2994
- socket.on("connect", () => {
2995
- console.log("[TourClient] Connected to tour agent server:", socket.id);
2996
- if (websiteId && userProfile) {
2997
- socket.emit("tour:init", { websiteId, userId: userProfile.userId, userType: userProfile.type });
2998
- }
2999
- });
3000
- socket.on("tour:server_state", (payload) => {
3001
- if (typeof payload?.runId === "number") {
3002
- runIdRef.current = payload.runId;
3003
- }
3004
- if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3005
- turnIdRef.current = payload.turnId ?? null;
2993
+ let createdSocket = null;
2994
+ const socket = io2(serverUrl, {
2995
+ path: "/socket.io",
2996
+ transports: resolveSocketIoTransports(serverUrl, "polling-first"),
2997
+ autoConnect: true,
2998
+ reconnection: true,
2999
+ reconnectionAttempts: 10,
3000
+ reconnectionDelay: 1e3
3001
+ });
3002
+ createdSocket = socket;
3003
+ socketRef.current = socket;
3004
+ socket.on("connect", () => {
3005
+ console.log("[TourClient] Connected to tour agent server:", socket.id);
3006
+ const profile = userProfileRef.current;
3007
+ if (websiteId && profile?.userId && socket.connected) {
3008
+ socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3009
+ }
3010
+ });
3011
+ socket.on("tour:server_state", (payload) => {
3012
+ if (typeof payload?.runId === "number") {
3013
+ runIdRef.current = payload.runId;
3014
+ }
3015
+ if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3016
+ turnIdRef.current = payload.turnId ?? null;
3017
+ }
3018
+ setServerState(payload);
3019
+ });
3020
+ socket.on("tour:command_cancel", (payload) => {
3021
+ console.log("[TourClient] Received command_cancel:", payload);
3022
+ if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3023
+ activeCommandBatchIdRef.current = null;
3024
+ activeExecutionTokenRef.current++;
3025
+ commandInFlightRef.current = false;
3026
+ setPlaybackState("idle");
3027
+ if (typeof window !== "undefined" && window.speechSynthesis) {
3028
+ window.speechSynthesis.cancel();
3006
3029
  }
3007
- setServerState(payload);
3008
- });
3009
- socket.on("tour:command_cancel", (payload) => {
3010
- console.log("[TourClient] Received command_cancel:", payload);
3011
- if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3030
+ }
3031
+ });
3032
+ socket.on("tour:command", async (payload) => {
3033
+ const emitIfOpen = (ev, data) => {
3034
+ if (socketRef.current !== socket) return;
3035
+ if (!socket.connected) return;
3036
+ socket.emit(ev, data);
3037
+ };
3038
+ console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3039
+ runCleanup(pendingManualWaitCleanupRef.current);
3040
+ pendingManualWaitCleanupRef.current = null;
3041
+ if (voiceInputResolveRef.current) {
3042
+ const resolvePendingVoiceInput = voiceInputResolveRef.current;
3043
+ voiceInputResolveRef.current = null;
3044
+ resolvePendingVoiceInput("");
3045
+ }
3046
+ setPlaybackState("executing");
3047
+ commandInFlightRef.current = true;
3048
+ const commandBatchId = payload.commandBatchId ?? null;
3049
+ turnIdRef.current = payload.turnId ?? turnIdRef.current;
3050
+ const clearCommandBatchId = () => {
3051
+ if (activeCommandBatchIdRef.current === commandBatchId) {
3012
3052
  activeCommandBatchIdRef.current = null;
3013
- activeExecutionTokenRef.current++;
3014
- commandInFlightRef.current = false;
3015
- setPlaybackState("idle");
3016
- if (typeof window !== "undefined" && window.speechSynthesis) {
3017
- window.speechSynthesis.cancel();
3018
- }
3019
3053
  }
3020
- });
3021
- socket.on("tour:command", async (payload) => {
3022
- console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3023
- runCleanup(pendingManualWaitCleanupRef.current);
3024
- pendingManualWaitCleanupRef.current = null;
3025
- if (voiceInputResolveRef.current) {
3026
- const resolvePendingVoiceInput = voiceInputResolveRef.current;
3027
- voiceInputResolveRef.current = null;
3028
- resolvePendingVoiceInput("");
3029
- }
3030
- setPlaybackState("executing");
3031
- commandInFlightRef.current = true;
3032
- const commandBatchId = payload.commandBatchId ?? null;
3033
- turnIdRef.current = payload.turnId ?? turnIdRef.current;
3034
- const clearCommandBatchId = () => {
3035
- if (activeCommandBatchIdRef.current === commandBatchId) {
3036
- activeCommandBatchIdRef.current = null;
3054
+ };
3055
+ activeCommandBatchIdRef.current = commandBatchId;
3056
+ const executionToken = ++activeExecutionTokenRef.current;
3057
+ const activeTourId = tourRef.current?.id;
3058
+ const activePreviewRunId = previewRunIdRef.current;
3059
+ if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3060
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3061
+ stepOrder: payload.stepIndex,
3062
+ eventType: "command_batch_received",
3063
+ payload: {
3064
+ commandBatchId,
3065
+ commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3066
+ },
3067
+ currentStepOrder: payload.stepIndex
3068
+ });
3069
+ }
3070
+ if (typeof payload.stepIndex === "number") {
3071
+ const prevStep = stepIndexRef.current;
3072
+ stepIndexRef.current = payload.stepIndex;
3073
+ setCurrentStepIndex(payload.stepIndex);
3074
+ if (payload.stepIndex !== prevStep) {
3075
+ const tour = tourRef.current;
3076
+ const total = tour?.steps?.length ?? 0;
3077
+ if (tour && total > 0) {
3078
+ onStepChangeRef.current?.(payload.stepIndex, total, tour);
3037
3079
  }
3038
- };
3039
- activeCommandBatchIdRef.current = commandBatchId;
3040
- const executionToken = ++activeExecutionTokenRef.current;
3041
- const activeTourId = tourRef.current?.id;
3042
- const activePreviewRunId = previewRunIdRef.current;
3043
- if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3080
+ }
3081
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3044
3082
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3045
3083
  stepOrder: payload.stepIndex,
3046
- eventType: "command_batch_received",
3084
+ eventType: "step_started",
3047
3085
  payload: {
3048
- commandBatchId,
3049
- commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3086
+ previousStepOrder: prevStep,
3087
+ stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3050
3088
  },
3051
3089
  currentStepOrder: payload.stepIndex
3052
3090
  });
3053
3091
  }
3054
- if (typeof payload.stepIndex === "number") {
3055
- const prevStep = stepIndexRef.current;
3056
- stepIndexRef.current = payload.stepIndex;
3057
- setCurrentStepIndex(payload.stepIndex);
3058
- if (payload.stepIndex !== prevStep) {
3059
- const tour = tourRef.current;
3060
- const total = tour?.steps?.length ?? 0;
3061
- if (tour && total > 0) {
3062
- onStepChangeRef.current?.(payload.stepIndex, total, tour);
3063
- }
3064
- }
3065
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3066
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3067
- stepOrder: payload.stepIndex,
3068
- eventType: "step_started",
3069
- payload: {
3070
- previousStepOrder: prevStep,
3071
- stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3072
- },
3073
- currentStepOrder: payload.stepIndex
3074
- });
3075
- }
3076
- }
3077
- if (!payload.commands || !Array.isArray(payload.commands)) {
3078
- console.warn("[TourClient] Payload commands is not an array:", payload);
3079
- commandInFlightRef.current = false;
3080
- socket.emit("tour:action_result", {
3081
- success: false,
3082
- reason: "invalid_commands",
3083
- commandBatchId,
3084
- runId: runIdRef.current,
3085
- turnId: turnIdRef.current
3086
- });
3087
- clearCommandBatchId();
3088
- return;
3092
+ }
3093
+ if (!payload.commands || !Array.isArray(payload.commands)) {
3094
+ console.warn("[TourClient] Payload commands is not an array:", payload);
3095
+ commandInFlightRef.current = false;
3096
+ emitIfOpen("tour:action_result", {
3097
+ success: false,
3098
+ reason: "invalid_commands",
3099
+ commandBatchId,
3100
+ runId: runIdRef.current,
3101
+ turnId: turnIdRef.current
3102
+ });
3103
+ clearCommandBatchId();
3104
+ return;
3105
+ }
3106
+ let shouldWait = false;
3107
+ const results = [];
3108
+ let batchPreferredWaitTarget = null;
3109
+ const assertNotInterrupted = () => {
3110
+ if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3111
+ const error = new Error("interrupted");
3112
+ error.code = "INTERRUPTED";
3113
+ throw error;
3089
3114
  }
3090
- let shouldWait = false;
3091
- const results = [];
3092
- let batchPreferredWaitTarget = null;
3093
- const assertNotInterrupted = () => {
3094
- if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3095
- const error = new Error("interrupted");
3096
- error.code = "INTERRUPTED";
3097
- throw error;
3098
- }
3099
- };
3100
- const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3101
- const fallbackHints = fallbackStep?.element ?? null;
3102
- return retryLookup({
3103
- timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3104
- pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3105
- onRetry: () => {
3106
- assertNotInterrupted();
3107
- },
3108
- resolve: async () => {
3109
- let targetEl = null;
3110
- if (params.uid) {
3111
- const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
3112
- targetEl = getElementByUid(params.uid);
3113
- }
3114
- if (!targetEl) {
3115
- targetEl = resolveElementFromHints({
3116
- fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3117
- testId: params.testId ?? fallbackHints?.testId,
3118
- textContaining: params.textContaining ?? fallbackHints?.textContaining
3119
- }, fallbackStep, tagStore);
3120
- }
3121
- return targetEl;
3122
- }
3123
- });
3124
- };
3125
- const executeOne = async (action) => {
3126
- assertNotInterrupted();
3127
- console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3128
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3129
- const executeTimeline = async (params = {}) => {
3130
- const segments = Array.isArray(params.segments) ? params.segments : [];
3131
- for (let index = 0; index < segments.length; index += 1) {
3132
- assertNotInterrupted();
3133
- const segment = segments[index];
3134
- const segmentText = (segment?.text ?? "").trim();
3135
- const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3136
- const events = Array.isArray(segment?.events) ? segment.events : [];
3137
- if (segmentDelayMs > 0) {
3138
- await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3139
- }
3140
- if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3141
- const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3142
- if (!gateTarget) {
3143
- throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3144
- }
3145
- await waitForUserAction(gateTarget, segment.gate.event);
3146
- }
3147
- if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3148
- showCaption(segmentText);
3149
- }
3150
- const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3151
- const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3152
- prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3153
- onNearEnd: nextSegmentText ? () => {
3154
- void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3155
- } : void 0
3156
- }) : Promise.resolve();
3157
- const eventsPromise = (async () => {
3158
- for (const event of events) {
3159
- assertNotInterrupted();
3160
- const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3161
- if (delayMs > 0) {
3162
- await new Promise((resolve) => setTimeout(resolve, delayMs));
3163
- }
3164
- if (event?.action) {
3165
- await executeOne(event.action);
3166
- }
3167
- }
3168
- })();
3169
- await Promise.all([speechPromise, eventsPromise]);
3170
- }
3171
- if (params.removeHighlightAtEnd !== false) {
3172
- removeHighlight();
3173
- }
3174
- if (showCaptionsRef.current && reviewModeRef.current) {
3175
- removeCaption();
3176
- }
3177
- return !!params.waitForInput;
3178
- };
3179
- if (action.type === "speak") {
3180
- const text = action.params?.text ?? "";
3181
- if (!text.trim()) {
3182
- return { result: null };
3115
+ };
3116
+ const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3117
+ const fallbackHints = fallbackStep?.element ?? null;
3118
+ return retryLookup({
3119
+ timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3120
+ pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3121
+ onRetry: () => {
3122
+ assertNotInterrupted();
3123
+ },
3124
+ resolve: async () => {
3125
+ let targetEl = null;
3126
+ if (params.uid) {
3127
+ const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
3128
+ targetEl = getElementByUid(params.uid);
3183
3129
  }
3184
- if (showCaptionsRef.current && reviewModeRef.current) {
3185
- showCaption(text);
3130
+ if (!targetEl) {
3131
+ targetEl = resolveElementFromHints({
3132
+ fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3133
+ testId: params.testId ?? fallbackHints?.testId,
3134
+ textContaining: params.textContaining ?? fallbackHints?.textContaining
3135
+ }, fallbackStep, tagStore);
3186
3136
  }
3187
- const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3188
- // Schedule narration immediately so the client keeps its speculative,
3189
- // low-latency behavior, but defer batch completion until playback settles.
3190
- waitForCompletion: false,
3191
- interrupt: action.params?.interrupt
3192
- });
3193
- return { result: text, settlePromise };
3137
+ return targetEl;
3194
3138
  }
3195
- if (action.type === "play_timeline") {
3196
- const timelineShouldWait = await executeTimeline(action.params);
3197
- if (timelineShouldWait) shouldWait = true;
3198
- return { result: "timeline_executed" };
3199
- }
3200
- if (action.type === "highlight_element") {
3201
- const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3202
- if (resolvedEl) {
3203
- if (isEditableWaitTarget(resolvedEl)) {
3204
- batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3139
+ });
3140
+ };
3141
+ const executeOne = async (action) => {
3142
+ assertNotInterrupted();
3143
+ console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3144
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3145
+ const executeTimeline = async (params = {}) => {
3146
+ const segments = Array.isArray(params.segments) ? params.segments : [];
3147
+ for (let index = 0; index < segments.length; index += 1) {
3148
+ assertNotInterrupted();
3149
+ const segment = segments[index];
3150
+ const segmentText = (segment?.text ?? "").trim();
3151
+ const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3152
+ const events = Array.isArray(segment?.events) ? segment.events : [];
3153
+ if (segmentDelayMs > 0) {
3154
+ await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3155
+ }
3156
+ if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3157
+ const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3158
+ if (!gateTarget) {
3159
+ throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3205
3160
  }
3206
- showHighlight(resolvedEl, action.params?.label || action.label);
3207
- resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3208
- return { result: "highlighted" };
3161
+ await waitForUserAction(gateTarget, segment.gate.event);
3209
3162
  }
3210
- if (!action.params?.optional) {
3211
- throw new Error(
3212
- `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3213
- );
3163
+ if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3164
+ showCaption(segmentText);
3214
3165
  }
3215
- return { result: "highlight_optional_miss" };
3166
+ const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3167
+ const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3168
+ prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3169
+ onNearEnd: nextSegmentText ? () => {
3170
+ void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3171
+ } : void 0
3172
+ }) : Promise.resolve();
3173
+ const eventsPromise = (async () => {
3174
+ for (const event of events) {
3175
+ assertNotInterrupted();
3176
+ const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3177
+ if (delayMs > 0) {
3178
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
3179
+ }
3180
+ if (event?.action) {
3181
+ await executeOne(event.action);
3182
+ }
3183
+ }
3184
+ })();
3185
+ await Promise.all([speechPromise, eventsPromise]);
3216
3186
  }
3217
- if (action.type === "remove_highlight") {
3187
+ if (params.removeHighlightAtEnd !== false) {
3218
3188
  removeHighlight();
3219
- return { result: "highlight_removed" };
3220
3189
  }
3221
- if (action.type === "click_element") {
3222
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3223
- if (!targetEl) {
3224
- if (action.params?.optional) return { result: "click_optional_miss" };
3225
- throw new Error(
3226
- `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3227
- );
3228
- }
3229
- removeHighlight();
3230
- await performInteractiveClick(targetEl);
3231
- const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
3232
- await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3233
- return { result: "clicked" };
3190
+ if (showCaptionsRef.current && reviewModeRef.current) {
3191
+ removeCaption();
3234
3192
  }
3235
- if (action.type === "fill_input") {
3236
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3237
- if (!targetEl) {
3238
- throw new Error(
3239
- `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3240
- );
3241
- }
3242
- const value = typeof action.params?.value === "string" ? action.params.value : "";
3243
- batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3244
- await performInteractiveFill(targetEl, value);
3245
- return { result: value };
3193
+ return !!params.waitForInput;
3194
+ };
3195
+ if (action.type === "speak") {
3196
+ const text = action.params?.text ?? "";
3197
+ if (!text.trim()) {
3198
+ return { result: null };
3246
3199
  }
3247
- if (action.type === "take_screenshot") {
3248
- const html2canvasModule = await import("html2canvas");
3249
- const html2canvas2 = html2canvasModule.default;
3250
- const canvas = await html2canvas2(document.body, {
3251
- useCORS: true,
3252
- allowTaint: true,
3253
- scale: Math.min(window.devicePixelRatio, 2),
3254
- width: window.innerWidth,
3255
- height: window.innerHeight,
3256
- x: window.scrollX,
3257
- y: window.scrollY,
3258
- logging: false
3259
- });
3260
- return { result: canvas.toDataURL("image/png") };
3200
+ if (showCaptionsRef.current && reviewModeRef.current) {
3201
+ showCaption(text);
3261
3202
  }
3262
- if (action.type === "navigate_to_url") {
3263
- const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3264
- if (!nextUrl) {
3265
- throw new Error("navigate_to_url missing url");
3203
+ const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3204
+ // Schedule narration immediately so the client keeps its speculative,
3205
+ // low-latency behavior, but defer batch completion until playback settles.
3206
+ waitForCompletion: false,
3207
+ interrupt: action.params?.interrupt
3208
+ });
3209
+ return { result: text, settlePromise };
3210
+ }
3211
+ if (action.type === "play_timeline") {
3212
+ const timelineShouldWait = await executeTimeline(action.params);
3213
+ if (timelineShouldWait) shouldWait = true;
3214
+ return { result: "timeline_executed" };
3215
+ }
3216
+ if (action.type === "highlight_element") {
3217
+ const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3218
+ if (resolvedEl) {
3219
+ if (isEditableWaitTarget(resolvedEl)) {
3220
+ batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3266
3221
  }
3267
- await navigateToTourUrl(nextUrl);
3268
- const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
3269
- await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3270
- return { result: nextUrl };
3222
+ showHighlight(resolvedEl, action.params?.label || action.label);
3223
+ resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3224
+ return { result: "highlighted" };
3271
3225
  }
3272
- if (action.type === "execute_agent_action") {
3273
- const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3274
- if (!agentSocketId) {
3275
- throw new Error("No socketId available for execute_agent_action");
3276
- }
3277
- const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3278
- const res = await fetch(url, {
3279
- method: "POST",
3280
- headers: { "Content-Type": "application/json" },
3281
- body: JSON.stringify({
3282
- command: action.params?.command,
3283
- socketId: agentSocketId
3284
- })
3285
- });
3286
- if (!res.ok) {
3287
- throw new Error(`execute_agent_action failed: ${await res.text()}`);
3288
- }
3289
- if (action.params?.wait !== false) {
3290
- await res.json();
3291
- }
3292
- const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3293
- await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
3294
- await syncAOM();
3295
- return { result: action.params?.command ?? "executed" };
3226
+ if (!action.params?.optional) {
3227
+ throw new Error(
3228
+ `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3229
+ );
3296
3230
  }
3297
- if (action.type === "wait_for_user_action") {
3298
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3299
- if (!targetEl) {
3300
- throw new Error("wait_for_user_action target not found");
3301
- }
3302
- const eventName = action.params?.event ?? "click";
3303
- await waitForUserAction(targetEl, eventName);
3304
- return { result: `waited_for_${eventName}` };
3231
+ return { result: "highlight_optional_miss" };
3232
+ }
3233
+ if (action.type === "remove_highlight") {
3234
+ removeHighlight();
3235
+ return { result: "highlight_removed" };
3236
+ }
3237
+ if (action.type === "click_element") {
3238
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3239
+ if (!targetEl) {
3240
+ if (action.params?.optional) return { result: "click_optional_miss" };
3241
+ throw new Error(
3242
+ `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3243
+ );
3305
3244
  }
3306
- if (action.type === "wait_for_input") {
3307
- shouldWait = true;
3308
- return { result: "waiting_for_input" };
3245
+ removeHighlight();
3246
+ await performInteractiveClick(targetEl);
3247
+ const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
3248
+ await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3249
+ return { result: "clicked" };
3250
+ }
3251
+ if (action.type === "fill_input") {
3252
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3253
+ if (!targetEl) {
3254
+ throw new Error(
3255
+ `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3256
+ );
3309
3257
  }
3310
- if (action.type === "end_tour") {
3311
- handleTourEnd();
3312
- return { result: "ended" };
3258
+ const value = typeof action.params?.value === "string" ? action.params.value : "";
3259
+ batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3260
+ await performInteractiveFill(targetEl, value);
3261
+ return { result: value };
3262
+ }
3263
+ if (action.type === "take_screenshot") {
3264
+ const html2canvasModule = await import("html2canvas");
3265
+ const html2canvas2 = html2canvasModule.default;
3266
+ const canvas = await html2canvas2(document.body, {
3267
+ useCORS: true,
3268
+ allowTaint: true,
3269
+ scale: Math.min(window.devicePixelRatio, 2),
3270
+ width: window.innerWidth,
3271
+ height: window.innerHeight,
3272
+ x: window.scrollX,
3273
+ y: window.scrollY,
3274
+ logging: false
3275
+ });
3276
+ return { result: canvas.toDataURL("image/png") };
3277
+ }
3278
+ if (action.type === "navigate_to_url") {
3279
+ const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3280
+ if (!nextUrl) {
3281
+ throw new Error("navigate_to_url missing url");
3313
3282
  }
3314
- console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3315
- return { result: "unknown_action_skipped" };
3316
- };
3317
- try {
3318
- const resultsBuffer = new Array(payload.commands.length);
3319
- const pendingUIActions = [];
3320
- for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3321
- const command = payload.commands[commandIndex];
3322
- assertNotInterrupted();
3323
- const isTerminal = isTerminalAction(command);
3324
- if (isTerminal) {
3325
- await Promise.all(pendingUIActions);
3326
- pendingUIActions.length = 0;
3327
- }
3328
- const executionPromise = (async () => {
3329
- const execution = await executeOne(command);
3330
- await execution.settlePromise;
3331
- resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3332
- })();
3333
- if (isTerminal) {
3334
- await executionPromise;
3335
- } else {
3336
- pendingUIActions.push(executionPromise);
3337
- }
3283
+ await navigateToTourUrl(nextUrl);
3284
+ const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
3285
+ await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3286
+ return { result: nextUrl };
3287
+ }
3288
+ if (action.type === "execute_agent_action") {
3289
+ const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3290
+ if (!agentSocketId) {
3291
+ throw new Error("No socketId available for execute_agent_action");
3338
3292
  }
3339
- await Promise.all(pendingUIActions);
3340
- resultsBuffer.forEach((res) => {
3341
- if (res) results.push(res);
3293
+ const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3294
+ const res = await fetch(url, {
3295
+ method: "POST",
3296
+ headers: { "Content-Type": "application/json" },
3297
+ body: JSON.stringify({
3298
+ command: action.params?.command,
3299
+ socketId: agentSocketId
3300
+ })
3342
3301
  });
3302
+ if (!res.ok) {
3303
+ throw new Error(`execute_agent_action failed: ${await res.text()}`);
3304
+ }
3305
+ if (action.params?.wait !== false) {
3306
+ await res.json();
3307
+ }
3308
+ const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3309
+ await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
3343
3310
  await syncAOM();
3344
- } catch (err) {
3345
- commandInFlightRef.current = false;
3346
- const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3347
- if (interrupted) {
3348
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3349
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3350
- stepOrder: stepIndexRef.current,
3351
- eventType: "command_batch_interrupted",
3352
- payload: {
3353
- commandBatchId,
3354
- partialResults: results
3355
- },
3356
- currentStepOrder: stepIndexRef.current
3357
- });
3358
- }
3359
- socket.emit("tour:action_result", {
3360
- success: true,
3361
- interrupted: true,
3362
- results,
3363
- commandBatchId,
3364
- runId: runIdRef.current,
3365
- turnId: turnIdRef.current
3366
- });
3367
- clearCommandBatchId();
3368
- return;
3311
+ return { result: action.params?.command ?? "executed" };
3312
+ }
3313
+ if (action.type === "wait_for_user_action") {
3314
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3315
+ if (!targetEl) {
3316
+ throw new Error("wait_for_user_action target not found");
3317
+ }
3318
+ const eventName = action.params?.event ?? "click";
3319
+ await waitForUserAction(targetEl, eventName);
3320
+ return { result: `waited_for_${eventName}` };
3321
+ }
3322
+ if (action.type === "wait_for_input") {
3323
+ shouldWait = true;
3324
+ return { result: "waiting_for_input" };
3325
+ }
3326
+ if (action.type === "end_tour") {
3327
+ handleTourEnd();
3328
+ return { result: "ended" };
3329
+ }
3330
+ console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3331
+ return { result: "unknown_action_skipped" };
3332
+ };
3333
+ try {
3334
+ const resultsBuffer = new Array(payload.commands.length);
3335
+ const pendingUIActions = [];
3336
+ for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3337
+ const command = payload.commands[commandIndex];
3338
+ assertNotInterrupted();
3339
+ const isTerminal = isTerminalAction(command);
3340
+ if (isTerminal) {
3341
+ await Promise.all(pendingUIActions);
3342
+ pendingUIActions.length = 0;
3369
3343
  }
3370
- console.error("[TourClient] Command batch execution failed:", err);
3344
+ const executionPromise = (async () => {
3345
+ const execution = await executeOne(command);
3346
+ await execution.settlePromise;
3347
+ resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3348
+ })();
3349
+ if (isTerminal) {
3350
+ await executionPromise;
3351
+ } else {
3352
+ pendingUIActions.push(executionPromise);
3353
+ }
3354
+ }
3355
+ await Promise.all(pendingUIActions);
3356
+ resultsBuffer.forEach((res) => {
3357
+ if (res) results.push(res);
3358
+ });
3359
+ await syncAOM();
3360
+ } catch (err) {
3361
+ commandInFlightRef.current = false;
3362
+ const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3363
+ if (interrupted) {
3371
3364
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3372
3365
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3373
3366
  stepOrder: stepIndexRef.current,
3374
- eventType: "command_batch_failed",
3367
+ eventType: "command_batch_interrupted",
3375
3368
  payload: {
3376
3369
  commandBatchId,
3377
- error: String(err),
3378
3370
  partialResults: results
3379
3371
  },
3380
- status: "active",
3381
3372
  currentStepOrder: stepIndexRef.current
3382
3373
  });
3383
3374
  }
3384
- socket.emit("tour:action_result", {
3385
- success: false,
3386
- reason: "execution_error",
3387
- error: String(err),
3375
+ emitIfOpen("tour:action_result", {
3376
+ success: true,
3377
+ interrupted: true,
3388
3378
  results,
3389
3379
  commandBatchId,
3390
3380
  runId: runIdRef.current,
@@ -3393,108 +3383,82 @@ function useTourPlayback({
3393
3383
  clearCommandBatchId();
3394
3384
  return;
3395
3385
  }
3396
- commandInFlightRef.current = false;
3397
- if (shouldWait && !skipRequestedRef.current) {
3398
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3399
- const waitCondition = currentStep?.onboarding?.waitCondition;
3400
- const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3401
- const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3402
- const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3403
- let manualWaitPromise = null;
3404
- let manualWaitKind = null;
3405
- const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3406
- const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3407
- runCleanup(pendingManualWaitCleanupRef.current);
3408
- pendingManualWaitCleanupRef.current = null;
3409
- if (waitTargetHints) {
3410
- let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3411
- if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3412
- manualWaitTarget = preferredWaitTarget;
3413
- console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3414
- }
3415
- if (manualWaitTarget) {
3416
- const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3417
- manualWaitPromise = manualWait.promise;
3418
- manualWaitKind = manualWait.kind;
3419
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3420
- }
3386
+ console.error("[TourClient] Command batch execution failed:", err);
3387
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3388
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3389
+ stepOrder: stepIndexRef.current,
3390
+ eventType: "command_batch_failed",
3391
+ payload: {
3392
+ commandBatchId,
3393
+ error: String(err),
3394
+ partialResults: results
3395
+ },
3396
+ status: "active",
3397
+ currentStepOrder: stepIndexRef.current
3398
+ });
3399
+ }
3400
+ emitIfOpen("tour:action_result", {
3401
+ success: false,
3402
+ reason: "execution_error",
3403
+ error: String(err),
3404
+ results,
3405
+ commandBatchId,
3406
+ runId: runIdRef.current,
3407
+ turnId: turnIdRef.current
3408
+ });
3409
+ clearCommandBatchId();
3410
+ return;
3411
+ }
3412
+ commandInFlightRef.current = false;
3413
+ if (shouldWait && !skipRequestedRef.current) {
3414
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3415
+ const waitCondition = currentStep?.onboarding?.waitCondition;
3416
+ const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3417
+ const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3418
+ const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3419
+ let manualWaitPromise = null;
3420
+ let manualWaitKind = null;
3421
+ const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3422
+ const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3423
+ runCleanup(pendingManualWaitCleanupRef.current);
3424
+ pendingManualWaitCleanupRef.current = null;
3425
+ if (waitTargetHints) {
3426
+ let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3427
+ if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3428
+ manualWaitTarget = preferredWaitTarget;
3429
+ console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3421
3430
  }
3422
- if (!manualWaitPromise && preferredWaitTarget) {
3423
- const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3431
+ if (manualWaitTarget) {
3432
+ const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3424
3433
  manualWaitPromise = manualWait.promise;
3425
3434
  manualWaitKind = manualWait.kind;
3426
3435
  pendingManualWaitCleanupRef.current = manualWait.cleanup;
3427
- console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3428
3436
  }
3429
- if (!manualWaitPromise && inputLikeWait) {
3430
- const firstInput = document.querySelector(
3431
- 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3432
- );
3433
- if (firstInput) {
3434
- const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3435
- manualWaitPromise = manualWait.promise;
3436
- manualWaitKind = manualWait.kind;
3437
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3438
- console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3439
- }
3440
- }
3441
- setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3442
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3443
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3444
- stepOrder: stepIndexRef.current,
3445
- eventType: "waiting_for_input",
3446
- payload: {
3447
- commandBatchId,
3448
- results
3449
- },
3450
- currentStepOrder: stepIndexRef.current
3451
- });
3437
+ }
3438
+ if (!manualWaitPromise && preferredWaitTarget) {
3439
+ const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3440
+ manualWaitPromise = manualWait.promise;
3441
+ manualWaitKind = manualWait.kind;
3442
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3443
+ console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3444
+ }
3445
+ if (!manualWaitPromise && inputLikeWait) {
3446
+ const firstInput = document.querySelector(
3447
+ 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3448
+ );
3449
+ if (firstInput) {
3450
+ const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3451
+ manualWaitPromise = manualWait.promise;
3452
+ manualWaitKind = manualWait.kind;
3453
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3454
+ console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3452
3455
  }
3453
- socket.emit("tour:action_result", {
3454
- success: true,
3455
- waitingForInput: true,
3456
- results,
3457
- commandBatchId,
3458
- runId: runIdRef.current,
3459
- turnId: turnIdRef.current
3460
- });
3461
- clearCommandBatchId();
3462
- const voiceOrTextWaitPromise = new Promise((resolve) => {
3463
- if (pendingInputBufRef.current) {
3464
- const flushed = pendingInputBufRef.current;
3465
- pendingInputBufRef.current = null;
3466
- resolve(flushed);
3467
- return;
3468
- }
3469
- voiceInputResolveRef.current = (text) => {
3470
- voiceInputResolveRef.current = null;
3471
- resolve(text);
3472
- };
3473
- });
3474
- Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3475
- runCleanup(pendingManualWaitCleanupRef.current);
3476
- pendingManualWaitCleanupRef.current = null;
3477
- voiceInputResolveRef.current = null;
3478
- setPlaybackState("executing");
3479
- const transcript = userText.trim();
3480
- if (!transcript) {
3481
- return;
3482
- }
3483
- const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3484
- await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
3485
- await syncAOM();
3486
- socket.emit("tour:user_input", {
3487
- transcript,
3488
- runId: runIdRef.current,
3489
- turnId: turnIdRef.current
3490
- });
3491
- });
3492
- return;
3493
3456
  }
3457
+ setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3494
3458
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3495
3459
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3496
3460
  stepOrder: stepIndexRef.current,
3497
- eventType: "command_batch_completed",
3461
+ eventType: "waiting_for_input",
3498
3462
  payload: {
3499
3463
  commandBatchId,
3500
3464
  results
@@ -3502,101 +3466,169 @@ function useTourPlayback({
3502
3466
  currentStepOrder: stepIndexRef.current
3503
3467
  });
3504
3468
  }
3505
- socket.emit("tour:action_result", {
3469
+ emitIfOpen("tour:action_result", {
3506
3470
  success: true,
3471
+ waitingForInput: true,
3507
3472
  results,
3508
3473
  commandBatchId,
3509
3474
  runId: runIdRef.current,
3510
3475
  turnId: turnIdRef.current
3511
3476
  });
3512
3477
  clearCommandBatchId();
3513
- });
3514
- socket.on("tour:start", async (tourData) => {
3515
- if (isActiveRef.current) return;
3516
- runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3517
- const tour = tourData.tourContext ?? tourRef.current;
3518
- if (tour?.type && tour.type !== experienceType) {
3519
- console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${experienceType})`);
3520
- return;
3521
- }
3522
- skipRequestedRef.current = false;
3523
- const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3524
- isActiveRef.current = true;
3525
- setIsActive(true);
3526
- setActiveTour(tour ?? null);
3527
- tourRef.current = tour ?? null;
3528
- setTotalSteps(total);
3529
- stepIndexRef.current = 0;
3530
- setCurrentStepIndex(0);
3531
- setPlaybackState("intro");
3532
- if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3533
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3534
- stepOrder: 0,
3535
- eventType: "tour_started",
3536
- payload: {
3537
- totalSteps: total,
3538
- source: "sdk_test_preview"
3539
- },
3540
- currentStepOrder: 0
3478
+ const voiceOrTextWaitPromise = new Promise((resolve) => {
3479
+ if (pendingInputBufRef.current) {
3480
+ const flushed = pendingInputBufRef.current;
3481
+ pendingInputBufRef.current = null;
3482
+ resolve(flushed);
3483
+ return;
3484
+ }
3485
+ voiceInputResolveRef.current = (text) => {
3486
+ voiceInputResolveRef.current = null;
3487
+ resolve(text);
3488
+ };
3489
+ });
3490
+ Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3491
+ runCleanup(pendingManualWaitCleanupRef.current);
3492
+ pendingManualWaitCleanupRef.current = null;
3493
+ voiceInputResolveRef.current = null;
3494
+ setPlaybackState("executing");
3495
+ const transcript = userText.trim();
3496
+ if (!transcript) {
3497
+ return;
3498
+ }
3499
+ const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3500
+ await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
3501
+ await syncAOM();
3502
+ emitIfOpen("tour:user_input", {
3503
+ transcript,
3504
+ runId: runIdRef.current,
3505
+ turnId: turnIdRef.current
3541
3506
  });
3542
- }
3543
- try {
3544
- const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3545
- const aom = generateMinifiedAOM2();
3507
+ });
3508
+ return;
3509
+ }
3510
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3511
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3512
+ stepOrder: stepIndexRef.current,
3513
+ eventType: "command_batch_completed",
3514
+ payload: {
3515
+ commandBatchId,
3516
+ results
3517
+ },
3518
+ currentStepOrder: stepIndexRef.current
3519
+ });
3520
+ }
3521
+ emitIfOpen("tour:action_result", {
3522
+ success: true,
3523
+ results,
3524
+ commandBatchId,
3525
+ runId: runIdRef.current,
3526
+ turnId: turnIdRef.current
3527
+ });
3528
+ clearCommandBatchId();
3529
+ });
3530
+ socket.on("tour:start", async (tourData) => {
3531
+ if (isActiveRef.current) return;
3532
+ runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3533
+ const tour = tourData.tourContext ?? tourRef.current;
3534
+ const expType = experienceTypeRef.current;
3535
+ if (tour?.type && tour.type !== expType) {
3536
+ console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3537
+ return;
3538
+ }
3539
+ skipRequestedRef.current = false;
3540
+ const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3541
+ isActiveRef.current = true;
3542
+ setIsActive(true);
3543
+ setActiveTour(tour ?? null);
3544
+ tourRef.current = tour ?? null;
3545
+ setTotalSteps(total);
3546
+ stepIndexRef.current = 0;
3547
+ setCurrentStepIndex(0);
3548
+ setPlaybackState("intro");
3549
+ if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3550
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3551
+ stepOrder: 0,
3552
+ eventType: "tour_started",
3553
+ payload: {
3554
+ totalSteps: total,
3555
+ source: "sdk_test_preview"
3556
+ },
3557
+ currentStepOrder: 0
3558
+ });
3559
+ }
3560
+ try {
3561
+ const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3562
+ const aom = generateMinifiedAOM2();
3563
+ if (socketRef.current === socket && socket.connected) {
3546
3564
  socket.emit("tour:sync_dom", {
3547
3565
  url: window.location.pathname + window.location.search + window.location.hash,
3548
3566
  aom: aom.nodes,
3549
3567
  domSummary: captureDomSummary()
3550
3568
  });
3551
- } catch (e) {
3552
- console.warn("[TourClient] Initial DOM sync failed:", e);
3553
3569
  }
3554
- });
3555
- socket.on("tour:update", (payload) => {
3556
- const updatedTour = payload?.tourContext;
3557
- if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3558
- return;
3559
- }
3560
- tourRef.current = updatedTour;
3561
- setActiveTour(updatedTour);
3562
- const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3563
- setTotalSteps(nextTotal);
3564
- if (typeof payload.currentStepIndex === "number") {
3565
- const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3566
- stepIndexRef.current = clampedStepIndex;
3567
- setCurrentStepIndex(clampedStepIndex);
3568
- if (nextTotal > 0) {
3569
- onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3570
- }
3570
+ } catch (e) {
3571
+ console.warn("[TourClient] Initial DOM sync failed:", e);
3572
+ }
3573
+ });
3574
+ socket.on("tour:update", (payload) => {
3575
+ const updatedTour = payload?.tourContext;
3576
+ if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3577
+ return;
3578
+ }
3579
+ tourRef.current = updatedTour;
3580
+ setActiveTour(updatedTour);
3581
+ const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3582
+ setTotalSteps(nextTotal);
3583
+ if (typeof payload.currentStepIndex === "number") {
3584
+ const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3585
+ stepIndexRef.current = clampedStepIndex;
3586
+ setCurrentStepIndex(clampedStepIndex);
3587
+ if (nextTotal > 0) {
3588
+ onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3571
3589
  }
3572
- });
3573
- socket.on("tour:end", () => {
3574
- setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3575
- handleTourEnd();
3576
- });
3577
- socket.on("tour:debug_log", (entry) => {
3578
- const isDev = devMode || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3579
- if (isDev) {
3580
- console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3581
- if (typeof window !== "undefined") {
3582
- window.dispatchEvent(new CustomEvent("modelnex-debug", {
3583
- detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3584
- }));
3585
- }
3590
+ }
3591
+ });
3592
+ socket.on("tour:end", () => {
3593
+ setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3594
+ handleTourEnd();
3595
+ });
3596
+ socket.on("tour:debug_log", (entry) => {
3597
+ const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3598
+ if (isDev) {
3599
+ console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3600
+ if (typeof window !== "undefined") {
3601
+ window.dispatchEvent(new CustomEvent("modelnex-debug", {
3602
+ detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3603
+ }));
3586
3604
  }
3587
- });
3588
- console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devMode || process.env.NODE_ENV === "development");
3605
+ }
3589
3606
  });
3607
+ console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3590
3608
  return () => {
3591
- if (socketRef.current) {
3592
- socketRef.current.disconnect();
3593
- socketRef.current = null;
3609
+ const toClose = createdSocket ?? socketRef.current;
3610
+ if (toClose) {
3611
+ toClose.disconnect();
3594
3612
  }
3613
+ createdSocket = null;
3614
+ socketRef.current = null;
3595
3615
  setServerState(null);
3596
3616
  runIdRef.current = null;
3597
3617
  turnIdRef.current = null;
3598
3618
  };
3599
- }, [serverUrl, websiteId, userProfile, disabled, devMode, experienceType]);
3619
+ }, [serverUrl, websiteId, disabled]);
3620
+ useEffect11(() => {
3621
+ if (disabled) return;
3622
+ const s = socketRef.current;
3623
+ const profile = userProfile;
3624
+ if (!s?.connected || !websiteId || !profile?.userId) return;
3625
+ const timer = setTimeout(() => {
3626
+ if (s.connected) {
3627
+ s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3628
+ }
3629
+ }, 150);
3630
+ return () => clearTimeout(timer);
3631
+ }, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
3600
3632
  useEffect11(() => {
3601
3633
  if (!showCaptions || !isReviewMode) {
3602
3634
  removeCaption();
@@ -3620,11 +3652,13 @@ function useTourPlayback({
3620
3652
  if (!socketRef.current?.connected || !isActiveRef.current) return;
3621
3653
  const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3622
3654
  const aom = generateMinifiedAOM2();
3623
- socketRef.current.emit("tour:sync_dom", {
3624
- url: window.location.pathname + window.location.search + window.location.hash,
3625
- aom: aom.nodes,
3626
- domSummary: captureDomSummary()
3627
- });
3655
+ if (socketRef.current?.connected) {
3656
+ socketRef.current.emit("tour:sync_dom", {
3657
+ url: window.location.pathname + window.location.search + window.location.hash,
3658
+ aom: aom.nodes,
3659
+ domSummary: captureDomSummary()
3660
+ });
3661
+ }
3628
3662
  }, []);
3629
3663
  const interruptExecution = useCallback7((transcript) => {
3630
3664
  if (!socketRef.current?.connected || !isActiveRef.current) return false;
@@ -3635,21 +3669,23 @@ function useTourPlayback({
3635
3669
  removeHighlight();
3636
3670
  removeCaption();
3637
3671
  voice.stopSpeaking();
3638
- socketRef.current.emit("tour:action_result", {
3639
- success: true,
3640
- interrupted: true,
3641
- transcript,
3642
- commandBatchId: activeCommandBatchIdRef.current,
3643
- runId: runIdRef.current,
3644
- turnId: turnIdRef.current
3645
- });
3646
- activeCommandBatchIdRef.current = null;
3647
- socketRef.current.emit("tour:user_input", {
3648
- transcript,
3649
- interrupted: true,
3650
- runId: runIdRef.current,
3651
- turnId: turnIdRef.current
3652
- });
3672
+ if (socketRef.current?.connected) {
3673
+ socketRef.current.emit("tour:action_result", {
3674
+ success: true,
3675
+ interrupted: true,
3676
+ transcript,
3677
+ commandBatchId: activeCommandBatchIdRef.current,
3678
+ runId: runIdRef.current,
3679
+ turnId: turnIdRef.current
3680
+ });
3681
+ activeCommandBatchIdRef.current = null;
3682
+ socketRef.current.emit("tour:user_input", {
3683
+ transcript,
3684
+ interrupted: true,
3685
+ runId: runIdRef.current,
3686
+ turnId: turnIdRef.current
3687
+ });
3688
+ }
3653
3689
  setPlaybackState("thinking");
3654
3690
  return true;
3655
3691
  }, [voice]);
@@ -3767,11 +3803,13 @@ function useTourPlayback({
3767
3803
  previewRunIdRef.current = null;
3768
3804
  }
3769
3805
  tourRef.current = tour;
3770
- socketRef.current.emit("tour:request_start", {
3771
- tourId: tour.id,
3772
- previewRunId: previewRunIdRef.current,
3773
- tourContext: tour
3774
- });
3806
+ if (socketRef.current?.connected) {
3807
+ socketRef.current.emit("tour:request_start", {
3808
+ tourId: tour.id,
3809
+ previewRunId: previewRunIdRef.current,
3810
+ tourContext: tour
3811
+ });
3812
+ }
3775
3813
  }, [serverUrl, websiteId]);
3776
3814
  useEffect11(() => {
3777
3815
  if (!enableAutoDiscovery) return;
@@ -5219,7 +5257,7 @@ function useVoice(serverUrl) {
5219
5257
  (async () => {
5220
5258
  try {
5221
5259
  const ioModule = await import("socket.io-client");
5222
- const io2 = ioModule.default || ioModule;
5260
+ const io3 = ioModule.default || ioModule;
5223
5261
  const stream = await navigator.mediaDevices.getUserMedia({
5224
5262
  audio: {
5225
5263
  echoCancellation: true,
@@ -5241,7 +5279,7 @@ function useVoice(serverUrl) {
5241
5279
  audioBitsPerSecond: 128e3
5242
5280
  });
5243
5281
  mediaRecorderRef.current = recorder;
5244
- const socket = io2(serverUrl, {
5282
+ const socket = io3(serverUrl, {
5245
5283
  path: "/socket.io",
5246
5284
  transports: resolveSocketIoTransports(serverUrl, "websocket-first")
5247
5285
  });
@@ -10761,7 +10799,7 @@ var ModelNexProvider = ({
10761
10799
  socketId,
10762
10800
  devMode
10763
10801
  }),
10764
- [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
10802
+ [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
10765
10803
  );
10766
10804
  return React8.createElement(
10767
10805
  ModelNexContext.Provider,