@modelnex/sdk 0.5.15 → 0.5.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -2416,6 +2416,7 @@ function isTourEligible(tour, userProfile) {
2416
2416
 
2417
2417
  // src/hooks/useTourPlayback.ts
2418
2418
  import { useState as useState7, useRef as useRef8, useCallback as useCallback7, useEffect as useEffect11, useContext as useContext4 } from "react";
2419
+ import { io as io2 } from "socket.io-client";
2419
2420
 
2420
2421
  // src/utils/retryLookup.ts
2421
2422
  var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -2989,629 +2990,622 @@ function useTourPlayback({
2989
2990
  useEffect11(() => {
2990
2991
  if (disabled) return;
2991
2992
  if (typeof window === "undefined") return;
2992
- let cancelled = false;
2993
2993
  let createdSocket = null;
2994
- import("socket.io-client").then((ioModule) => {
2995
- if (cancelled) return;
2996
- const io2 = ioModule.default || ioModule;
2997
- const socket = io2(serverUrl, {
2998
- path: "/socket.io",
2999
- // standard
3000
- transports: resolveSocketIoTransports(serverUrl, "polling-first")
3001
- });
3002
- if (cancelled) {
3003
- socket.disconnect();
3004
- return;
2994
+ const socket = io2(serverUrl, {
2995
+ path: "/socket.io",
2996
+ transports: resolveSocketIoTransports(serverUrl, "polling-first"),
2997
+ autoConnect: true,
2998
+ reconnection: true,
2999
+ reconnectionAttempts: 10,
3000
+ reconnectionDelay: 1e3
3001
+ });
3002
+ createdSocket = socket;
3003
+ socketRef.current = socket;
3004
+ socket.on("connect", () => {
3005
+ console.log("[TourClient] Connected to tour agent server:", socket.id);
3006
+ const profile = userProfileRef.current;
3007
+ if (websiteId && profile?.userId && socket.connected) {
3008
+ socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3005
3009
  }
3006
- createdSocket = socket;
3007
- socketRef.current = socket;
3008
- socket.on("connect", () => {
3009
- console.log("[TourClient] Connected to tour agent server:", socket.id);
3010
- const profile = userProfileRef.current;
3011
- if (websiteId && profile?.userId) {
3012
- socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3013
- }
3014
- });
3015
- socket.on("tour:server_state", (payload) => {
3016
- if (typeof payload?.runId === "number") {
3017
- runIdRef.current = payload.runId;
3018
- }
3019
- if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3020
- turnIdRef.current = payload.turnId ?? null;
3010
+ });
3011
+ socket.on("tour:server_state", (payload) => {
3012
+ if (typeof payload?.runId === "number") {
3013
+ runIdRef.current = payload.runId;
3014
+ }
3015
+ if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3016
+ turnIdRef.current = payload.turnId ?? null;
3017
+ }
3018
+ setServerState(payload);
3019
+ });
3020
+ socket.on("tour:command_cancel", (payload) => {
3021
+ console.log("[TourClient] Received command_cancel:", payload);
3022
+ if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3023
+ activeCommandBatchIdRef.current = null;
3024
+ activeExecutionTokenRef.current++;
3025
+ commandInFlightRef.current = false;
3026
+ setPlaybackState("idle");
3027
+ if (typeof window !== "undefined" && window.speechSynthesis) {
3028
+ window.speechSynthesis.cancel();
3021
3029
  }
3022
- setServerState(payload);
3023
- });
3024
- socket.on("tour:command_cancel", (payload) => {
3025
- console.log("[TourClient] Received command_cancel:", payload);
3026
- if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3030
+ }
3031
+ });
3032
+ socket.on("tour:command", async (payload) => {
3033
+ const emitIfOpen = (ev, data) => {
3034
+ if (socketRef.current !== socket) return;
3035
+ if (!socket.connected) return;
3036
+ socket.emit(ev, data);
3037
+ };
3038
+ console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3039
+ runCleanup(pendingManualWaitCleanupRef.current);
3040
+ pendingManualWaitCleanupRef.current = null;
3041
+ if (voiceInputResolveRef.current) {
3042
+ const resolvePendingVoiceInput = voiceInputResolveRef.current;
3043
+ voiceInputResolveRef.current = null;
3044
+ resolvePendingVoiceInput("");
3045
+ }
3046
+ setPlaybackState("executing");
3047
+ commandInFlightRef.current = true;
3048
+ const commandBatchId = payload.commandBatchId ?? null;
3049
+ turnIdRef.current = payload.turnId ?? turnIdRef.current;
3050
+ const clearCommandBatchId = () => {
3051
+ if (activeCommandBatchIdRef.current === commandBatchId) {
3027
3052
  activeCommandBatchIdRef.current = null;
3028
- activeExecutionTokenRef.current++;
3029
- commandInFlightRef.current = false;
3030
- setPlaybackState("idle");
3031
- if (typeof window !== "undefined" && window.speechSynthesis) {
3032
- window.speechSynthesis.cancel();
3033
- }
3034
3053
  }
3035
- });
3036
- socket.on("tour:command", async (payload) => {
3037
- const emitIfOpen = (ev, data) => {
3038
- if (socketRef.current !== socket) return;
3039
- if (!socket.connected) return;
3040
- socket.emit(ev, data);
3041
- };
3042
- console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3043
- runCleanup(pendingManualWaitCleanupRef.current);
3044
- pendingManualWaitCleanupRef.current = null;
3045
- if (voiceInputResolveRef.current) {
3046
- const resolvePendingVoiceInput = voiceInputResolveRef.current;
3047
- voiceInputResolveRef.current = null;
3048
- resolvePendingVoiceInput("");
3049
- }
3050
- setPlaybackState("executing");
3051
- commandInFlightRef.current = true;
3052
- const commandBatchId = payload.commandBatchId ?? null;
3053
- turnIdRef.current = payload.turnId ?? turnIdRef.current;
3054
- const clearCommandBatchId = () => {
3055
- if (activeCommandBatchIdRef.current === commandBatchId) {
3056
- activeCommandBatchIdRef.current = null;
3054
+ };
3055
+ activeCommandBatchIdRef.current = commandBatchId;
3056
+ const executionToken = ++activeExecutionTokenRef.current;
3057
+ const activeTourId = tourRef.current?.id;
3058
+ const activePreviewRunId = previewRunIdRef.current;
3059
+ if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3060
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3061
+ stepOrder: payload.stepIndex,
3062
+ eventType: "command_batch_received",
3063
+ payload: {
3064
+ commandBatchId,
3065
+ commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3066
+ },
3067
+ currentStepOrder: payload.stepIndex
3068
+ });
3069
+ }
3070
+ if (typeof payload.stepIndex === "number") {
3071
+ const prevStep = stepIndexRef.current;
3072
+ stepIndexRef.current = payload.stepIndex;
3073
+ setCurrentStepIndex(payload.stepIndex);
3074
+ if (payload.stepIndex !== prevStep) {
3075
+ const tour = tourRef.current;
3076
+ const total = tour?.steps?.length ?? 0;
3077
+ if (tour && total > 0) {
3078
+ onStepChangeRef.current?.(payload.stepIndex, total, tour);
3057
3079
  }
3058
- };
3059
- activeCommandBatchIdRef.current = commandBatchId;
3060
- const executionToken = ++activeExecutionTokenRef.current;
3061
- const activeTourId = tourRef.current?.id;
3062
- const activePreviewRunId = previewRunIdRef.current;
3063
- if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3080
+ }
3081
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3064
3082
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3065
3083
  stepOrder: payload.stepIndex,
3066
- eventType: "command_batch_received",
3084
+ eventType: "step_started",
3067
3085
  payload: {
3068
- commandBatchId,
3069
- commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3086
+ previousStepOrder: prevStep,
3087
+ stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3070
3088
  },
3071
3089
  currentStepOrder: payload.stepIndex
3072
3090
  });
3073
3091
  }
3074
- if (typeof payload.stepIndex === "number") {
3075
- const prevStep = stepIndexRef.current;
3076
- stepIndexRef.current = payload.stepIndex;
3077
- setCurrentStepIndex(payload.stepIndex);
3078
- if (payload.stepIndex !== prevStep) {
3079
- const tour = tourRef.current;
3080
- const total = tour?.steps?.length ?? 0;
3081
- if (tour && total > 0) {
3082
- onStepChangeRef.current?.(payload.stepIndex, total, tour);
3083
- }
3084
- }
3085
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3086
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3087
- stepOrder: payload.stepIndex,
3088
- eventType: "step_started",
3089
- payload: {
3090
- previousStepOrder: prevStep,
3091
- stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3092
- },
3093
- currentStepOrder: payload.stepIndex
3094
- });
3095
- }
3096
- }
3097
- if (!payload.commands || !Array.isArray(payload.commands)) {
3098
- console.warn("[TourClient] Payload commands is not an array:", payload);
3099
- commandInFlightRef.current = false;
3100
- emitIfOpen("tour:action_result", {
3101
- success: false,
3102
- reason: "invalid_commands",
3103
- commandBatchId,
3104
- runId: runIdRef.current,
3105
- turnId: turnIdRef.current
3106
- });
3107
- clearCommandBatchId();
3108
- return;
3092
+ }
3093
+ if (!payload.commands || !Array.isArray(payload.commands)) {
3094
+ console.warn("[TourClient] Payload commands is not an array:", payload);
3095
+ commandInFlightRef.current = false;
3096
+ emitIfOpen("tour:action_result", {
3097
+ success: false,
3098
+ reason: "invalid_commands",
3099
+ commandBatchId,
3100
+ runId: runIdRef.current,
3101
+ turnId: turnIdRef.current
3102
+ });
3103
+ clearCommandBatchId();
3104
+ return;
3105
+ }
3106
+ let shouldWait = false;
3107
+ const results = [];
3108
+ let batchPreferredWaitTarget = null;
3109
+ const assertNotInterrupted = () => {
3110
+ if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3111
+ const error = new Error("interrupted");
3112
+ error.code = "INTERRUPTED";
3113
+ throw error;
3109
3114
  }
3110
- let shouldWait = false;
3111
- const results = [];
3112
- let batchPreferredWaitTarget = null;
3113
- const assertNotInterrupted = () => {
3114
- if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3115
- const error = new Error("interrupted");
3116
- error.code = "INTERRUPTED";
3117
- throw error;
3118
- }
3119
- };
3120
- const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3121
- const fallbackHints = fallbackStep?.element ?? null;
3122
- return retryLookup({
3123
- timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3124
- pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3125
- onRetry: () => {
3126
- assertNotInterrupted();
3127
- },
3128
- resolve: async () => {
3129
- let targetEl = null;
3130
- if (params.uid) {
3131
- const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
3132
- targetEl = getElementByUid(params.uid);
3133
- }
3134
- if (!targetEl) {
3135
- targetEl = resolveElementFromHints({
3136
- fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3137
- testId: params.testId ?? fallbackHints?.testId,
3138
- textContaining: params.textContaining ?? fallbackHints?.textContaining
3139
- }, fallbackStep, tagStore);
3140
- }
3141
- return targetEl;
3142
- }
3143
- });
3144
- };
3145
- const executeOne = async (action) => {
3146
- assertNotInterrupted();
3147
- console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3148
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3149
- const executeTimeline = async (params = {}) => {
3150
- const segments = Array.isArray(params.segments) ? params.segments : [];
3151
- for (let index = 0; index < segments.length; index += 1) {
3152
- assertNotInterrupted();
3153
- const segment = segments[index];
3154
- const segmentText = (segment?.text ?? "").trim();
3155
- const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3156
- const events = Array.isArray(segment?.events) ? segment.events : [];
3157
- if (segmentDelayMs > 0) {
3158
- await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3159
- }
3160
- if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3161
- const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3162
- if (!gateTarget) {
3163
- throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3164
- }
3165
- await waitForUserAction(gateTarget, segment.gate.event);
3166
- }
3167
- if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3168
- showCaption(segmentText);
3169
- }
3170
- const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3171
- const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3172
- prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3173
- onNearEnd: nextSegmentText ? () => {
3174
- void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3175
- } : void 0
3176
- }) : Promise.resolve();
3177
- const eventsPromise = (async () => {
3178
- for (const event of events) {
3179
- assertNotInterrupted();
3180
- const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3181
- if (delayMs > 0) {
3182
- await new Promise((resolve) => setTimeout(resolve, delayMs));
3183
- }
3184
- if (event?.action) {
3185
- await executeOne(event.action);
3186
- }
3187
- }
3188
- })();
3189
- await Promise.all([speechPromise, eventsPromise]);
3190
- }
3191
- if (params.removeHighlightAtEnd !== false) {
3192
- removeHighlight();
3193
- }
3194
- if (showCaptionsRef.current && reviewModeRef.current) {
3195
- removeCaption();
3196
- }
3197
- return !!params.waitForInput;
3198
- };
3199
- if (action.type === "speak") {
3200
- const text = action.params?.text ?? "";
3201
- if (!text.trim()) {
3202
- return { result: null };
3115
+ };
3116
+ const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3117
+ const fallbackHints = fallbackStep?.element ?? null;
3118
+ return retryLookup({
3119
+ timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3120
+ pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3121
+ onRetry: () => {
3122
+ assertNotInterrupted();
3123
+ },
3124
+ resolve: async () => {
3125
+ let targetEl = null;
3126
+ if (params.uid) {
3127
+ const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
3128
+ targetEl = getElementByUid(params.uid);
3203
3129
  }
3204
- if (showCaptionsRef.current && reviewModeRef.current) {
3205
- showCaption(text);
3130
+ if (!targetEl) {
3131
+ targetEl = resolveElementFromHints({
3132
+ fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3133
+ testId: params.testId ?? fallbackHints?.testId,
3134
+ textContaining: params.textContaining ?? fallbackHints?.textContaining
3135
+ }, fallbackStep, tagStore);
3206
3136
  }
3207
- const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3208
- // Schedule narration immediately so the client keeps its speculative,
3209
- // low-latency behavior, but defer batch completion until playback settles.
3210
- waitForCompletion: false,
3211
- interrupt: action.params?.interrupt
3212
- });
3213
- return { result: text, settlePromise };
3214
- }
3215
- if (action.type === "play_timeline") {
3216
- const timelineShouldWait = await executeTimeline(action.params);
3217
- if (timelineShouldWait) shouldWait = true;
3218
- return { result: "timeline_executed" };
3137
+ return targetEl;
3219
3138
  }
3220
- if (action.type === "highlight_element") {
3221
- const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3222
- if (resolvedEl) {
3223
- if (isEditableWaitTarget(resolvedEl)) {
3224
- batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3139
+ });
3140
+ };
3141
+ const executeOne = async (action) => {
3142
+ assertNotInterrupted();
3143
+ console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3144
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3145
+ const executeTimeline = async (params = {}) => {
3146
+ const segments = Array.isArray(params.segments) ? params.segments : [];
3147
+ for (let index = 0; index < segments.length; index += 1) {
3148
+ assertNotInterrupted();
3149
+ const segment = segments[index];
3150
+ const segmentText = (segment?.text ?? "").trim();
3151
+ const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3152
+ const events = Array.isArray(segment?.events) ? segment.events : [];
3153
+ if (segmentDelayMs > 0) {
3154
+ await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3155
+ }
3156
+ if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3157
+ const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3158
+ if (!gateTarget) {
3159
+ throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3225
3160
  }
3226
- showHighlight(resolvedEl, action.params?.label || action.label);
3227
- resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3228
- return { result: "highlighted" };
3161
+ await waitForUserAction(gateTarget, segment.gate.event);
3229
3162
  }
3230
- if (!action.params?.optional) {
3231
- throw new Error(
3232
- `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3233
- );
3163
+ if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3164
+ showCaption(segmentText);
3234
3165
  }
3235
- return { result: "highlight_optional_miss" };
3166
+ const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3167
+ const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3168
+ prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3169
+ onNearEnd: nextSegmentText ? () => {
3170
+ void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3171
+ } : void 0
3172
+ }) : Promise.resolve();
3173
+ const eventsPromise = (async () => {
3174
+ for (const event of events) {
3175
+ assertNotInterrupted();
3176
+ const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3177
+ if (delayMs > 0) {
3178
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
3179
+ }
3180
+ if (event?.action) {
3181
+ await executeOne(event.action);
3182
+ }
3183
+ }
3184
+ })();
3185
+ await Promise.all([speechPromise, eventsPromise]);
3236
3186
  }
3237
- if (action.type === "remove_highlight") {
3187
+ if (params.removeHighlightAtEnd !== false) {
3238
3188
  removeHighlight();
3239
- return { result: "highlight_removed" };
3240
3189
  }
3241
- if (action.type === "click_element") {
3242
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3243
- if (!targetEl) {
3244
- if (action.params?.optional) return { result: "click_optional_miss" };
3245
- throw new Error(
3246
- `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3247
- );
3248
- }
3249
- removeHighlight();
3250
- await performInteractiveClick(targetEl);
3251
- const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
3252
- await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3253
- return { result: "clicked" };
3190
+ if (showCaptionsRef.current && reviewModeRef.current) {
3191
+ removeCaption();
3254
3192
  }
3255
- if (action.type === "fill_input") {
3256
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3257
- if (!targetEl) {
3258
- throw new Error(
3259
- `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3260
- );
3261
- }
3262
- const value = typeof action.params?.value === "string" ? action.params.value : "";
3263
- batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3264
- await performInteractiveFill(targetEl, value);
3265
- return { result: value };
3193
+ return !!params.waitForInput;
3194
+ };
3195
+ if (action.type === "speak") {
3196
+ const text = action.params?.text ?? "";
3197
+ if (!text.trim()) {
3198
+ return { result: null };
3266
3199
  }
3267
- if (action.type === "take_screenshot") {
3268
- const html2canvasModule = await import("html2canvas");
3269
- const html2canvas2 = html2canvasModule.default;
3270
- const canvas = await html2canvas2(document.body, {
3271
- useCORS: true,
3272
- allowTaint: true,
3273
- scale: Math.min(window.devicePixelRatio, 2),
3274
- width: window.innerWidth,
3275
- height: window.innerHeight,
3276
- x: window.scrollX,
3277
- y: window.scrollY,
3278
- logging: false
3279
- });
3280
- return { result: canvas.toDataURL("image/png") };
3200
+ if (showCaptionsRef.current && reviewModeRef.current) {
3201
+ showCaption(text);
3281
3202
  }
3282
- if (action.type === "navigate_to_url") {
3283
- const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3284
- if (!nextUrl) {
3285
- throw new Error("navigate_to_url missing url");
3203
+ const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3204
+ // Schedule narration immediately so the client keeps its speculative,
3205
+ // low-latency behavior, but defer batch completion until playback settles.
3206
+ waitForCompletion: false,
3207
+ interrupt: action.params?.interrupt
3208
+ });
3209
+ return { result: text, settlePromise };
3210
+ }
3211
+ if (action.type === "play_timeline") {
3212
+ const timelineShouldWait = await executeTimeline(action.params);
3213
+ if (timelineShouldWait) shouldWait = true;
3214
+ return { result: "timeline_executed" };
3215
+ }
3216
+ if (action.type === "highlight_element") {
3217
+ const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3218
+ if (resolvedEl) {
3219
+ if (isEditableWaitTarget(resolvedEl)) {
3220
+ batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3286
3221
  }
3287
- await navigateToTourUrl(nextUrl);
3288
- const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
3289
- await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3290
- return { result: nextUrl };
3222
+ showHighlight(resolvedEl, action.params?.label || action.label);
3223
+ resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3224
+ return { result: "highlighted" };
3291
3225
  }
3292
- if (action.type === "execute_agent_action") {
3293
- const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3294
- if (!agentSocketId) {
3295
- throw new Error("No socketId available for execute_agent_action");
3296
- }
3297
- const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3298
- const res = await fetch(url, {
3299
- method: "POST",
3300
- headers: { "Content-Type": "application/json" },
3301
- body: JSON.stringify({
3302
- command: action.params?.command,
3303
- socketId: agentSocketId
3304
- })
3305
- });
3306
- if (!res.ok) {
3307
- throw new Error(`execute_agent_action failed: ${await res.text()}`);
3308
- }
3309
- if (action.params?.wait !== false) {
3310
- await res.json();
3311
- }
3312
- const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3313
- await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
3314
- await syncAOM();
3315
- return { result: action.params?.command ?? "executed" };
3226
+ if (!action.params?.optional) {
3227
+ throw new Error(
3228
+ `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3229
+ );
3316
3230
  }
3317
- if (action.type === "wait_for_user_action") {
3318
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3319
- if (!targetEl) {
3320
- throw new Error("wait_for_user_action target not found");
3321
- }
3322
- const eventName = action.params?.event ?? "click";
3323
- await waitForUserAction(targetEl, eventName);
3324
- return { result: `waited_for_${eventName}` };
3231
+ return { result: "highlight_optional_miss" };
3232
+ }
3233
+ if (action.type === "remove_highlight") {
3234
+ removeHighlight();
3235
+ return { result: "highlight_removed" };
3236
+ }
3237
+ if (action.type === "click_element") {
3238
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3239
+ if (!targetEl) {
3240
+ if (action.params?.optional) return { result: "click_optional_miss" };
3241
+ throw new Error(
3242
+ `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3243
+ );
3325
3244
  }
3326
- if (action.type === "wait_for_input") {
3327
- shouldWait = true;
3328
- return { result: "waiting_for_input" };
3245
+ removeHighlight();
3246
+ await performInteractiveClick(targetEl);
3247
+ const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
3248
+ await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3249
+ return { result: "clicked" };
3250
+ }
3251
+ if (action.type === "fill_input") {
3252
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3253
+ if (!targetEl) {
3254
+ throw new Error(
3255
+ `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3256
+ );
3329
3257
  }
3330
- if (action.type === "end_tour") {
3331
- handleTourEnd();
3332
- return { result: "ended" };
3258
+ const value = typeof action.params?.value === "string" ? action.params.value : "";
3259
+ batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3260
+ await performInteractiveFill(targetEl, value);
3261
+ return { result: value };
3262
+ }
3263
+ if (action.type === "take_screenshot") {
3264
+ const html2canvasModule = await import("html2canvas");
3265
+ const html2canvas2 = html2canvasModule.default;
3266
+ const canvas = await html2canvas2(document.body, {
3267
+ useCORS: true,
3268
+ allowTaint: true,
3269
+ scale: Math.min(window.devicePixelRatio, 2),
3270
+ width: window.innerWidth,
3271
+ height: window.innerHeight,
3272
+ x: window.scrollX,
3273
+ y: window.scrollY,
3274
+ logging: false
3275
+ });
3276
+ return { result: canvas.toDataURL("image/png") };
3277
+ }
3278
+ if (action.type === "navigate_to_url") {
3279
+ const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3280
+ if (!nextUrl) {
3281
+ throw new Error("navigate_to_url missing url");
3333
3282
  }
3334
- console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3335
- return { result: "unknown_action_skipped" };
3336
- };
3337
- try {
3338
- const resultsBuffer = new Array(payload.commands.length);
3339
- const pendingUIActions = [];
3340
- for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3341
- const command = payload.commands[commandIndex];
3342
- assertNotInterrupted();
3343
- const isTerminal = isTerminalAction(command);
3344
- if (isTerminal) {
3345
- await Promise.all(pendingUIActions);
3346
- pendingUIActions.length = 0;
3347
- }
3348
- const executionPromise = (async () => {
3349
- const execution = await executeOne(command);
3350
- await execution.settlePromise;
3351
- resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3352
- })();
3353
- if (isTerminal) {
3354
- await executionPromise;
3355
- } else {
3356
- pendingUIActions.push(executionPromise);
3357
- }
3283
+ await navigateToTourUrl(nextUrl);
3284
+ const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
3285
+ await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3286
+ return { result: nextUrl };
3287
+ }
3288
+ if (action.type === "execute_agent_action") {
3289
+ const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3290
+ if (!agentSocketId) {
3291
+ throw new Error("No socketId available for execute_agent_action");
3358
3292
  }
3359
- await Promise.all(pendingUIActions);
3360
- resultsBuffer.forEach((res) => {
3361
- if (res) results.push(res);
3293
+ const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3294
+ const res = await fetch(url, {
3295
+ method: "POST",
3296
+ headers: { "Content-Type": "application/json" },
3297
+ body: JSON.stringify({
3298
+ command: action.params?.command,
3299
+ socketId: agentSocketId
3300
+ })
3362
3301
  });
3363
- await syncAOM();
3364
- } catch (err) {
3365
- commandInFlightRef.current = false;
3366
- const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3367
- if (interrupted) {
3368
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3369
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3370
- stepOrder: stepIndexRef.current,
3371
- eventType: "command_batch_interrupted",
3372
- payload: {
3373
- commandBatchId,
3374
- partialResults: results
3375
- },
3376
- currentStepOrder: stepIndexRef.current
3377
- });
3378
- }
3379
- emitIfOpen("tour:action_result", {
3380
- success: true,
3381
- interrupted: true,
3382
- results,
3383
- commandBatchId,
3384
- runId: runIdRef.current,
3385
- turnId: turnIdRef.current
3386
- });
3387
- clearCommandBatchId();
3388
- return;
3302
+ if (!res.ok) {
3303
+ throw new Error(`execute_agent_action failed: ${await res.text()}`);
3389
3304
  }
3390
- console.error("[TourClient] Command batch execution failed:", err);
3391
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3392
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3393
- stepOrder: stepIndexRef.current,
3394
- eventType: "command_batch_failed",
3395
- payload: {
3396
- commandBatchId,
3397
- error: String(err),
3398
- partialResults: results
3399
- },
3400
- status: "active",
3401
- currentStepOrder: stepIndexRef.current
3402
- });
3305
+ if (action.params?.wait !== false) {
3306
+ await res.json();
3403
3307
  }
3404
- emitIfOpen("tour:action_result", {
3405
- success: false,
3406
- reason: "execution_error",
3407
- error: String(err),
3408
- results,
3409
- commandBatchId,
3410
- runId: runIdRef.current,
3411
- turnId: turnIdRef.current
3412
- });
3413
- clearCommandBatchId();
3414
- return;
3308
+ const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3309
+ await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
3310
+ await syncAOM();
3311
+ return { result: action.params?.command ?? "executed" };
3415
3312
  }
3416
- commandInFlightRef.current = false;
3417
- if (shouldWait && !skipRequestedRef.current) {
3418
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3419
- const waitCondition = currentStep?.onboarding?.waitCondition;
3420
- const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3421
- const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3422
- const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3423
- let manualWaitPromise = null;
3424
- let manualWaitKind = null;
3425
- const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3426
- const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3427
- runCleanup(pendingManualWaitCleanupRef.current);
3428
- pendingManualWaitCleanupRef.current = null;
3429
- if (waitTargetHints) {
3430
- let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3431
- if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3432
- manualWaitTarget = preferredWaitTarget;
3433
- console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3434
- }
3435
- if (manualWaitTarget) {
3436
- const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3437
- manualWaitPromise = manualWait.promise;
3438
- manualWaitKind = manualWait.kind;
3439
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3440
- }
3313
+ if (action.type === "wait_for_user_action") {
3314
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3315
+ if (!targetEl) {
3316
+ throw new Error("wait_for_user_action target not found");
3441
3317
  }
3442
- if (!manualWaitPromise && preferredWaitTarget) {
3443
- const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3444
- manualWaitPromise = manualWait.promise;
3445
- manualWaitKind = manualWait.kind;
3446
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3447
- console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3318
+ const eventName = action.params?.event ?? "click";
3319
+ await waitForUserAction(targetEl, eventName);
3320
+ return { result: `waited_for_${eventName}` };
3321
+ }
3322
+ if (action.type === "wait_for_input") {
3323
+ shouldWait = true;
3324
+ return { result: "waiting_for_input" };
3325
+ }
3326
+ if (action.type === "end_tour") {
3327
+ handleTourEnd();
3328
+ return { result: "ended" };
3329
+ }
3330
+ console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3331
+ return { result: "unknown_action_skipped" };
3332
+ };
3333
+ try {
3334
+ const resultsBuffer = new Array(payload.commands.length);
3335
+ const pendingUIActions = [];
3336
+ for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3337
+ const command = payload.commands[commandIndex];
3338
+ assertNotInterrupted();
3339
+ const isTerminal = isTerminalAction(command);
3340
+ if (isTerminal) {
3341
+ await Promise.all(pendingUIActions);
3342
+ pendingUIActions.length = 0;
3448
3343
  }
3449
- if (!manualWaitPromise && inputLikeWait) {
3450
- const firstInput = document.querySelector(
3451
- 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3452
- );
3453
- if (firstInput) {
3454
- const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3455
- manualWaitPromise = manualWait.promise;
3456
- manualWaitKind = manualWait.kind;
3457
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3458
- console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3459
- }
3344
+ const executionPromise = (async () => {
3345
+ const execution = await executeOne(command);
3346
+ await execution.settlePromise;
3347
+ resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3348
+ })();
3349
+ if (isTerminal) {
3350
+ await executionPromise;
3351
+ } else {
3352
+ pendingUIActions.push(executionPromise);
3460
3353
  }
3461
- setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3354
+ }
3355
+ await Promise.all(pendingUIActions);
3356
+ resultsBuffer.forEach((res) => {
3357
+ if (res) results.push(res);
3358
+ });
3359
+ await syncAOM();
3360
+ } catch (err) {
3361
+ commandInFlightRef.current = false;
3362
+ const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3363
+ if (interrupted) {
3462
3364
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3463
3365
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3464
3366
  stepOrder: stepIndexRef.current,
3465
- eventType: "waiting_for_input",
3367
+ eventType: "command_batch_interrupted",
3466
3368
  payload: {
3467
3369
  commandBatchId,
3468
- results
3370
+ partialResults: results
3469
3371
  },
3470
3372
  currentStepOrder: stepIndexRef.current
3471
3373
  });
3472
3374
  }
3473
3375
  emitIfOpen("tour:action_result", {
3474
3376
  success: true,
3475
- waitingForInput: true,
3377
+ interrupted: true,
3476
3378
  results,
3477
3379
  commandBatchId,
3478
3380
  runId: runIdRef.current,
3479
3381
  turnId: turnIdRef.current
3480
3382
  });
3481
3383
  clearCommandBatchId();
3482
- const voiceOrTextWaitPromise = new Promise((resolve) => {
3483
- if (pendingInputBufRef.current) {
3484
- const flushed = pendingInputBufRef.current;
3485
- pendingInputBufRef.current = null;
3486
- resolve(flushed);
3487
- return;
3488
- }
3489
- voiceInputResolveRef.current = (text) => {
3490
- voiceInputResolveRef.current = null;
3491
- resolve(text);
3492
- };
3493
- });
3494
- Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3495
- runCleanup(pendingManualWaitCleanupRef.current);
3496
- pendingManualWaitCleanupRef.current = null;
3497
- voiceInputResolveRef.current = null;
3498
- setPlaybackState("executing");
3499
- const transcript = userText.trim();
3500
- if (!transcript) {
3501
- return;
3502
- }
3503
- const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3504
- await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
3505
- await syncAOM();
3506
- emitIfOpen("tour:user_input", {
3507
- transcript,
3508
- runId: runIdRef.current,
3509
- turnId: turnIdRef.current
3510
- });
3511
- });
3512
3384
  return;
3513
3385
  }
3386
+ console.error("[TourClient] Command batch execution failed:", err);
3514
3387
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3515
3388
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3516
3389
  stepOrder: stepIndexRef.current,
3517
- eventType: "command_batch_completed",
3390
+ eventType: "command_batch_failed",
3518
3391
  payload: {
3519
3392
  commandBatchId,
3520
- results
3393
+ error: String(err),
3394
+ partialResults: results
3521
3395
  },
3396
+ status: "active",
3522
3397
  currentStepOrder: stepIndexRef.current
3523
3398
  });
3524
3399
  }
3525
3400
  emitIfOpen("tour:action_result", {
3526
- success: true,
3401
+ success: false,
3402
+ reason: "execution_error",
3403
+ error: String(err),
3527
3404
  results,
3528
3405
  commandBatchId,
3529
3406
  runId: runIdRef.current,
3530
3407
  turnId: turnIdRef.current
3531
3408
  });
3532
3409
  clearCommandBatchId();
3533
- });
3534
- socket.on("tour:start", async (tourData) => {
3535
- if (isActiveRef.current) return;
3536
- runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3537
- const tour = tourData.tourContext ?? tourRef.current;
3538
- const expType = experienceTypeRef.current;
3539
- if (tour?.type && tour.type !== expType) {
3540
- console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3541
- return;
3410
+ return;
3411
+ }
3412
+ commandInFlightRef.current = false;
3413
+ if (shouldWait && !skipRequestedRef.current) {
3414
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3415
+ const waitCondition = currentStep?.onboarding?.waitCondition;
3416
+ const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3417
+ const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3418
+ const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3419
+ let manualWaitPromise = null;
3420
+ let manualWaitKind = null;
3421
+ const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3422
+ const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3423
+ runCleanup(pendingManualWaitCleanupRef.current);
3424
+ pendingManualWaitCleanupRef.current = null;
3425
+ if (waitTargetHints) {
3426
+ let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3427
+ if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3428
+ manualWaitTarget = preferredWaitTarget;
3429
+ console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3430
+ }
3431
+ if (manualWaitTarget) {
3432
+ const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3433
+ manualWaitPromise = manualWait.promise;
3434
+ manualWaitKind = manualWait.kind;
3435
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3436
+ }
3542
3437
  }
3543
- skipRequestedRef.current = false;
3544
- const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3545
- isActiveRef.current = true;
3546
- setIsActive(true);
3547
- setActiveTour(tour ?? null);
3548
- tourRef.current = tour ?? null;
3549
- setTotalSteps(total);
3550
- stepIndexRef.current = 0;
3551
- setCurrentStepIndex(0);
3552
- setPlaybackState("intro");
3553
- if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3554
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3555
- stepOrder: 0,
3556
- eventType: "tour_started",
3438
+ if (!manualWaitPromise && preferredWaitTarget) {
3439
+ const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3440
+ manualWaitPromise = manualWait.promise;
3441
+ manualWaitKind = manualWait.kind;
3442
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3443
+ console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3444
+ }
3445
+ if (!manualWaitPromise && inputLikeWait) {
3446
+ const firstInput = document.querySelector(
3447
+ 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3448
+ );
3449
+ if (firstInput) {
3450
+ const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3451
+ manualWaitPromise = manualWait.promise;
3452
+ manualWaitKind = manualWait.kind;
3453
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3454
+ console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3455
+ }
3456
+ }
3457
+ setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3458
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3459
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3460
+ stepOrder: stepIndexRef.current,
3461
+ eventType: "waiting_for_input",
3557
3462
  payload: {
3558
- totalSteps: total,
3559
- source: "sdk_test_preview"
3463
+ commandBatchId,
3464
+ results
3560
3465
  },
3561
- currentStepOrder: 0
3466
+ currentStepOrder: stepIndexRef.current
3562
3467
  });
3563
3468
  }
3564
- try {
3565
- const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3566
- const aom = generateMinifiedAOM2();
3567
- if (socketRef.current === socket && socket.connected) {
3568
- socket.emit("tour:sync_dom", {
3569
- url: window.location.pathname + window.location.search + window.location.hash,
3570
- aom: aom.nodes,
3571
- domSummary: captureDomSummary()
3572
- });
3469
+ emitIfOpen("tour:action_result", {
3470
+ success: true,
3471
+ waitingForInput: true,
3472
+ results,
3473
+ commandBatchId,
3474
+ runId: runIdRef.current,
3475
+ turnId: turnIdRef.current
3476
+ });
3477
+ clearCommandBatchId();
3478
+ const voiceOrTextWaitPromise = new Promise((resolve) => {
3479
+ if (pendingInputBufRef.current) {
3480
+ const flushed = pendingInputBufRef.current;
3481
+ pendingInputBufRef.current = null;
3482
+ resolve(flushed);
3483
+ return;
3573
3484
  }
3574
- } catch (e) {
3575
- console.warn("[TourClient] Initial DOM sync failed:", e);
3576
- }
3485
+ voiceInputResolveRef.current = (text) => {
3486
+ voiceInputResolveRef.current = null;
3487
+ resolve(text);
3488
+ };
3489
+ });
3490
+ Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3491
+ runCleanup(pendingManualWaitCleanupRef.current);
3492
+ pendingManualWaitCleanupRef.current = null;
3493
+ voiceInputResolveRef.current = null;
3494
+ setPlaybackState("executing");
3495
+ const transcript = userText.trim();
3496
+ if (!transcript) {
3497
+ return;
3498
+ }
3499
+ const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3500
+ await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
3501
+ await syncAOM();
3502
+ emitIfOpen("tour:user_input", {
3503
+ transcript,
3504
+ runId: runIdRef.current,
3505
+ turnId: turnIdRef.current
3506
+ });
3507
+ });
3508
+ return;
3509
+ }
3510
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3511
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3512
+ stepOrder: stepIndexRef.current,
3513
+ eventType: "command_batch_completed",
3514
+ payload: {
3515
+ commandBatchId,
3516
+ results
3517
+ },
3518
+ currentStepOrder: stepIndexRef.current
3519
+ });
3520
+ }
3521
+ emitIfOpen("tour:action_result", {
3522
+ success: true,
3523
+ results,
3524
+ commandBatchId,
3525
+ runId: runIdRef.current,
3526
+ turnId: turnIdRef.current
3577
3527
  });
3578
- socket.on("tour:update", (payload) => {
3579
- const updatedTour = payload?.tourContext;
3580
- if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3581
- return;
3528
+ clearCommandBatchId();
3529
+ });
3530
+ socket.on("tour:start", async (tourData) => {
3531
+ if (isActiveRef.current) return;
3532
+ runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3533
+ const tour = tourData.tourContext ?? tourRef.current;
3534
+ const expType = experienceTypeRef.current;
3535
+ if (tour?.type && tour.type !== expType) {
3536
+ console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3537
+ return;
3538
+ }
3539
+ skipRequestedRef.current = false;
3540
+ const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3541
+ isActiveRef.current = true;
3542
+ setIsActive(true);
3543
+ setActiveTour(tour ?? null);
3544
+ tourRef.current = tour ?? null;
3545
+ setTotalSteps(total);
3546
+ stepIndexRef.current = 0;
3547
+ setCurrentStepIndex(0);
3548
+ setPlaybackState("intro");
3549
+ if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3550
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3551
+ stepOrder: 0,
3552
+ eventType: "tour_started",
3553
+ payload: {
3554
+ totalSteps: total,
3555
+ source: "sdk_test_preview"
3556
+ },
3557
+ currentStepOrder: 0
3558
+ });
3559
+ }
3560
+ try {
3561
+ const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3562
+ const aom = generateMinifiedAOM2();
3563
+ if (socketRef.current === socket && socket.connected) {
3564
+ socket.emit("tour:sync_dom", {
3565
+ url: window.location.pathname + window.location.search + window.location.hash,
3566
+ aom: aom.nodes,
3567
+ domSummary: captureDomSummary()
3568
+ });
3582
3569
  }
3583
- tourRef.current = updatedTour;
3584
- setActiveTour(updatedTour);
3585
- const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3586
- setTotalSteps(nextTotal);
3587
- if (typeof payload.currentStepIndex === "number") {
3588
- const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3589
- stepIndexRef.current = clampedStepIndex;
3590
- setCurrentStepIndex(clampedStepIndex);
3591
- if (nextTotal > 0) {
3592
- onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3593
- }
3570
+ } catch (e) {
3571
+ console.warn("[TourClient] Initial DOM sync failed:", e);
3572
+ }
3573
+ });
3574
+ socket.on("tour:update", (payload) => {
3575
+ const updatedTour = payload?.tourContext;
3576
+ if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3577
+ return;
3578
+ }
3579
+ tourRef.current = updatedTour;
3580
+ setActiveTour(updatedTour);
3581
+ const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3582
+ setTotalSteps(nextTotal);
3583
+ if (typeof payload.currentStepIndex === "number") {
3584
+ const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3585
+ stepIndexRef.current = clampedStepIndex;
3586
+ setCurrentStepIndex(clampedStepIndex);
3587
+ if (nextTotal > 0) {
3588
+ onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3594
3589
  }
3595
- });
3596
- socket.on("tour:end", () => {
3597
- setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3598
- handleTourEnd();
3599
- });
3600
- socket.on("tour:debug_log", (entry) => {
3601
- const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3602
- if (isDev) {
3603
- console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3604
- if (typeof window !== "undefined") {
3605
- window.dispatchEvent(new CustomEvent("modelnex-debug", {
3606
- detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3607
- }));
3608
- }
3590
+ }
3591
+ });
3592
+ socket.on("tour:end", () => {
3593
+ setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3594
+ handleTourEnd();
3595
+ });
3596
+ socket.on("tour:debug_log", (entry) => {
3597
+ const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3598
+ if (isDev) {
3599
+ console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3600
+ if (typeof window !== "undefined") {
3601
+ window.dispatchEvent(new CustomEvent("modelnex-debug", {
3602
+ detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3603
+ }));
3609
3604
  }
3610
- });
3611
- console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3605
+ }
3612
3606
  });
3607
+ console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3613
3608
  return () => {
3614
- cancelled = true;
3615
3609
  const toClose = createdSocket ?? socketRef.current;
3616
3610
  if (toClose) {
3617
3611
  toClose.disconnect();
@@ -3628,7 +3622,12 @@ function useTourPlayback({
3628
3622
  const s = socketRef.current;
3629
3623
  const profile = userProfile;
3630
3624
  if (!s?.connected || !websiteId || !profile?.userId) return;
3631
- s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3625
+ const timer = setTimeout(() => {
3626
+ if (s.connected) {
3627
+ s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3628
+ }
3629
+ }, 150);
3630
+ return () => clearTimeout(timer);
3632
3631
  }, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
3633
3632
  useEffect11(() => {
3634
3633
  if (!showCaptions || !isReviewMode) {
@@ -3653,11 +3652,13 @@ function useTourPlayback({
3653
3652
  if (!socketRef.current?.connected || !isActiveRef.current) return;
3654
3653
  const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3655
3654
  const aom = generateMinifiedAOM2();
3656
- socketRef.current.emit("tour:sync_dom", {
3657
- url: window.location.pathname + window.location.search + window.location.hash,
3658
- aom: aom.nodes,
3659
- domSummary: captureDomSummary()
3660
- });
3655
+ if (socketRef.current?.connected) {
3656
+ socketRef.current.emit("tour:sync_dom", {
3657
+ url: window.location.pathname + window.location.search + window.location.hash,
3658
+ aom: aom.nodes,
3659
+ domSummary: captureDomSummary()
3660
+ });
3661
+ }
3661
3662
  }, []);
3662
3663
  const interruptExecution = useCallback7((transcript) => {
3663
3664
  if (!socketRef.current?.connected || !isActiveRef.current) return false;
@@ -3668,21 +3669,23 @@ function useTourPlayback({
3668
3669
  removeHighlight();
3669
3670
  removeCaption();
3670
3671
  voice.stopSpeaking();
3671
- socketRef.current.emit("tour:action_result", {
3672
- success: true,
3673
- interrupted: true,
3674
- transcript,
3675
- commandBatchId: activeCommandBatchIdRef.current,
3676
- runId: runIdRef.current,
3677
- turnId: turnIdRef.current
3678
- });
3679
- activeCommandBatchIdRef.current = null;
3680
- socketRef.current.emit("tour:user_input", {
3681
- transcript,
3682
- interrupted: true,
3683
- runId: runIdRef.current,
3684
- turnId: turnIdRef.current
3685
- });
3672
+ if (socketRef.current?.connected) {
3673
+ socketRef.current.emit("tour:action_result", {
3674
+ success: true,
3675
+ interrupted: true,
3676
+ transcript,
3677
+ commandBatchId: activeCommandBatchIdRef.current,
3678
+ runId: runIdRef.current,
3679
+ turnId: turnIdRef.current
3680
+ });
3681
+ activeCommandBatchIdRef.current = null;
3682
+ socketRef.current.emit("tour:user_input", {
3683
+ transcript,
3684
+ interrupted: true,
3685
+ runId: runIdRef.current,
3686
+ turnId: turnIdRef.current
3687
+ });
3688
+ }
3686
3689
  setPlaybackState("thinking");
3687
3690
  return true;
3688
3691
  }, [voice]);
@@ -3800,11 +3803,13 @@ function useTourPlayback({
3800
3803
  previewRunIdRef.current = null;
3801
3804
  }
3802
3805
  tourRef.current = tour;
3803
- socketRef.current.emit("tour:request_start", {
3804
- tourId: tour.id,
3805
- previewRunId: previewRunIdRef.current,
3806
- tourContext: tour
3807
- });
3806
+ if (socketRef.current?.connected) {
3807
+ socketRef.current.emit("tour:request_start", {
3808
+ tourId: tour.id,
3809
+ previewRunId: previewRunIdRef.current,
3810
+ tourContext: tour
3811
+ });
3812
+ }
3808
3813
  }, [serverUrl, websiteId]);
3809
3814
  useEffect11(() => {
3810
3815
  if (!enableAutoDiscovery) return;
@@ -5252,7 +5257,7 @@ function useVoice(serverUrl) {
5252
5257
  (async () => {
5253
5258
  try {
5254
5259
  const ioModule = await import("socket.io-client");
5255
- const io2 = ioModule.default || ioModule;
5260
+ const io3 = ioModule.default || ioModule;
5256
5261
  const stream = await navigator.mediaDevices.getUserMedia({
5257
5262
  audio: {
5258
5263
  echoCancellation: true,
@@ -5274,7 +5279,7 @@ function useVoice(serverUrl) {
5274
5279
  audioBitsPerSecond: 128e3
5275
5280
  });
5276
5281
  mediaRecorderRef.current = recorder;
5277
- const socket = io2(serverUrl, {
5282
+ const socket = io3(serverUrl, {
5278
5283
  path: "/socket.io",
5279
5284
  transports: resolveSocketIoTransports(serverUrl, "websocket-first")
5280
5285
  });
@@ -10794,7 +10799,7 @@ var ModelNexProvider = ({
10794
10799
  socketId,
10795
10800
  devMode
10796
10801
  }),
10797
- [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
10802
+ [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
10798
10803
  );
10799
10804
  return React8.createElement(
10800
10805
  ModelNexContext.Provider,