@modelnex/sdk 0.5.15 → 0.5.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -2438,6 +2438,76 @@ async function retryLookup({
2438
2438
  }
2439
2439
  }
2440
2440
 
2441
+ // src/utils/tour-socket-pool.ts
2442
+ import { io as io2 } from "socket.io-client";
2443
+ function isSocketWritable(socket) {
2444
+ if (!socket?.connected) return false;
2445
+ const engine = socket.io?.engine;
2446
+ if (!engine) return true;
2447
+ if (typeof engine.readyState === "string" && engine.readyState !== "open") return false;
2448
+ if (engine.transport && "writable" in engine.transport && engine.transport.writable === false) return false;
2449
+ return true;
2450
+ }
2451
+ function emitSocketEvent(socket, event, payload) {
2452
+ if (!isSocketWritable(socket)) return false;
2453
+ socket.emit(event, payload);
2454
+ return true;
2455
+ }
2456
+ function createTourSocketPool({
2457
+ createSocket = (serverUrl) => io2(serverUrl, {
2458
+ path: "/socket.io",
2459
+ transports: resolveSocketIoTransports(serverUrl, "polling-first"),
2460
+ autoConnect: true,
2461
+ reconnection: true,
2462
+ reconnectionAttempts: 10,
2463
+ reconnectionDelay: 1e3
2464
+ }),
2465
+ releaseDelayMs = 2500,
2466
+ scheduleRelease = (callback, delayMs) => setTimeout(callback, delayMs),
2467
+ cancelRelease = (timer) => clearTimeout(timer)
2468
+ } = {}) {
2469
+ const pooledSockets = /* @__PURE__ */ new Map();
2470
+ return {
2471
+ acquire(serverUrl) {
2472
+ const pooled = pooledSockets.get(serverUrl);
2473
+ if (pooled) {
2474
+ if (pooled.releaseTimer) {
2475
+ cancelRelease(pooled.releaseTimer);
2476
+ pooled.releaseTimer = null;
2477
+ }
2478
+ pooled.leases += 1;
2479
+ return pooled.socket;
2480
+ }
2481
+ const socket = createSocket(serverUrl);
2482
+ pooledSockets.set(serverUrl, {
2483
+ socket,
2484
+ leases: 1,
2485
+ releaseTimer: null
2486
+ });
2487
+ return socket;
2488
+ },
2489
+ release(serverUrl, socket) {
2490
+ const pooled = pooledSockets.get(serverUrl);
2491
+ if (!pooled || pooled.socket !== socket) return;
2492
+ pooled.leases = Math.max(0, pooled.leases - 1);
2493
+ if (pooled.leases > 0) return;
2494
+ pooled.releaseTimer = scheduleRelease(() => {
2495
+ const latest = pooledSockets.get(serverUrl);
2496
+ if (!latest || latest !== pooled || latest.leases > 0) return;
2497
+ latest.socket.disconnect();
2498
+ pooledSockets.delete(serverUrl);
2499
+ }, releaseDelayMs);
2500
+ },
2501
+ // Test-only introspection
2502
+ getSnapshot(serverUrl) {
2503
+ const pooled = pooledSockets.get(serverUrl);
2504
+ if (!pooled) return null;
2505
+ return { leases: pooled.leases, socket: pooled.socket };
2506
+ }
2507
+ };
2508
+ }
2509
+ var tourSocketPool = createTourSocketPool();
2510
+
2441
2511
  // src/hooks/useTourPlayback.ts
2442
2512
  function resolveElement(step) {
2443
2513
  const el = step.element;
@@ -2974,11 +3044,13 @@ function useTourPlayback({
2974
3044
  const socketRef = useRef8(null);
2975
3045
  const socketIdRef = useRef8(socketId);
2976
3046
  const commandUrlRef = useRef8(commandUrl);
3047
+ const websiteIdRef = useRef8(websiteId);
2977
3048
  const onStepChangeRef = useRef8(onStepChange);
2978
3049
  const isActiveRef = useRef8(false);
2979
3050
  const activeCommandBatchIdRef = useRef8(null);
2980
3051
  socketIdRef.current = socketId;
2981
3052
  commandUrlRef.current = commandUrl;
3053
+ websiteIdRef.current = websiteId;
2982
3054
  onStepChangeRef.current = onStepChange;
2983
3055
  isActiveRef.current = isActive;
2984
3056
  reviewModeRef.current = isReviewMode;
@@ -2989,646 +3061,646 @@ function useTourPlayback({
2989
3061
  useEffect11(() => {
2990
3062
  if (disabled) return;
2991
3063
  if (typeof window === "undefined") return;
2992
- let cancelled = false;
2993
- let createdSocket = null;
2994
- import("socket.io-client").then((ioModule) => {
2995
- if (cancelled) return;
2996
- const io2 = ioModule.default || ioModule;
2997
- const socket = io2(serverUrl, {
2998
- path: "/socket.io",
2999
- // standard
3000
- transports: resolveSocketIoTransports(serverUrl, "polling-first")
3001
- });
3002
- if (cancelled) {
3003
- socket.disconnect();
3004
- return;
3064
+ const socket = tourSocketPool.acquire(serverUrl);
3065
+ socketRef.current = socket;
3066
+ const handleConnect = () => {
3067
+ console.log("[TourClient] Connected to tour agent server:", socket.id);
3068
+ const profile = userProfileRef.current;
3069
+ const currentWebsiteId = websiteIdRef.current;
3070
+ if (currentWebsiteId && profile?.userId) {
3071
+ emitSocketEvent(socket, "tour:init", { websiteId: currentWebsiteId, userId: profile.userId, userType: profile.type });
3005
3072
  }
3006
- createdSocket = socket;
3007
- socketRef.current = socket;
3008
- socket.on("connect", () => {
3009
- console.log("[TourClient] Connected to tour agent server:", socket.id);
3010
- const profile = userProfileRef.current;
3011
- if (websiteId && profile?.userId) {
3012
- socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3013
- }
3014
- });
3015
- socket.on("tour:server_state", (payload) => {
3016
- if (typeof payload?.runId === "number") {
3017
- runIdRef.current = payload.runId;
3018
- }
3019
- if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3020
- turnIdRef.current = payload.turnId ?? null;
3073
+ };
3074
+ const handleServerState = (payload) => {
3075
+ if (typeof payload?.runId === "number") {
3076
+ runIdRef.current = payload.runId;
3077
+ }
3078
+ if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3079
+ turnIdRef.current = payload.turnId ?? null;
3080
+ }
3081
+ setServerState(payload);
3082
+ };
3083
+ const handleCommandCancel = (payload) => {
3084
+ console.log("[TourClient] Received command_cancel:", payload);
3085
+ if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3086
+ activeCommandBatchIdRef.current = null;
3087
+ activeExecutionTokenRef.current++;
3088
+ commandInFlightRef.current = false;
3089
+ setPlaybackState("idle");
3090
+ if (typeof window !== "undefined" && window.speechSynthesis) {
3091
+ window.speechSynthesis.cancel();
3021
3092
  }
3022
- setServerState(payload);
3023
- });
3024
- socket.on("tour:command_cancel", (payload) => {
3025
- console.log("[TourClient] Received command_cancel:", payload);
3026
- if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3093
+ }
3094
+ };
3095
+ const handleCommand = async (payload) => {
3096
+ const emitIfOpen = (ev, data) => {
3097
+ if (socketRef.current !== socket) return;
3098
+ emitSocketEvent(socket, ev, data);
3099
+ };
3100
+ console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3101
+ runCleanup(pendingManualWaitCleanupRef.current);
3102
+ pendingManualWaitCleanupRef.current = null;
3103
+ if (voiceInputResolveRef.current) {
3104
+ const resolvePendingVoiceInput = voiceInputResolveRef.current;
3105
+ voiceInputResolveRef.current = null;
3106
+ resolvePendingVoiceInput("");
3107
+ }
3108
+ setPlaybackState("executing");
3109
+ commandInFlightRef.current = true;
3110
+ const commandBatchId = payload.commandBatchId ?? null;
3111
+ turnIdRef.current = payload.turnId ?? turnIdRef.current;
3112
+ const clearCommandBatchId = () => {
3113
+ if (activeCommandBatchIdRef.current === commandBatchId) {
3027
3114
  activeCommandBatchIdRef.current = null;
3028
- activeExecutionTokenRef.current++;
3029
- commandInFlightRef.current = false;
3030
- setPlaybackState("idle");
3031
- if (typeof window !== "undefined" && window.speechSynthesis) {
3032
- window.speechSynthesis.cancel();
3033
- }
3034
- }
3035
- });
3036
- socket.on("tour:command", async (payload) => {
3037
- const emitIfOpen = (ev, data) => {
3038
- if (socketRef.current !== socket) return;
3039
- if (!socket.connected) return;
3040
- socket.emit(ev, data);
3041
- };
3042
- console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3043
- runCleanup(pendingManualWaitCleanupRef.current);
3044
- pendingManualWaitCleanupRef.current = null;
3045
- if (voiceInputResolveRef.current) {
3046
- const resolvePendingVoiceInput = voiceInputResolveRef.current;
3047
- voiceInputResolveRef.current = null;
3048
- resolvePendingVoiceInput("");
3049
3115
  }
3050
- setPlaybackState("executing");
3051
- commandInFlightRef.current = true;
3052
- const commandBatchId = payload.commandBatchId ?? null;
3053
- turnIdRef.current = payload.turnId ?? turnIdRef.current;
3054
- const clearCommandBatchId = () => {
3055
- if (activeCommandBatchIdRef.current === commandBatchId) {
3056
- activeCommandBatchIdRef.current = null;
3116
+ };
3117
+ activeCommandBatchIdRef.current = commandBatchId;
3118
+ const executionToken = ++activeExecutionTokenRef.current;
3119
+ const activeTourId = tourRef.current?.id;
3120
+ const activePreviewRunId = previewRunIdRef.current;
3121
+ if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3122
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3123
+ stepOrder: payload.stepIndex,
3124
+ eventType: "command_batch_received",
3125
+ payload: {
3126
+ commandBatchId,
3127
+ commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3128
+ },
3129
+ currentStepOrder: payload.stepIndex
3130
+ });
3131
+ }
3132
+ if (typeof payload.stepIndex === "number") {
3133
+ const prevStep = stepIndexRef.current;
3134
+ stepIndexRef.current = payload.stepIndex;
3135
+ setCurrentStepIndex(payload.stepIndex);
3136
+ if (payload.stepIndex !== prevStep) {
3137
+ const tour = tourRef.current;
3138
+ const total = tour?.steps?.length ?? 0;
3139
+ if (tour && total > 0) {
3140
+ onStepChangeRef.current?.(payload.stepIndex, total, tour);
3057
3141
  }
3058
- };
3059
- activeCommandBatchIdRef.current = commandBatchId;
3060
- const executionToken = ++activeExecutionTokenRef.current;
3061
- const activeTourId = tourRef.current?.id;
3062
- const activePreviewRunId = previewRunIdRef.current;
3063
- if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3142
+ }
3143
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3064
3144
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3065
3145
  stepOrder: payload.stepIndex,
3066
- eventType: "command_batch_received",
3146
+ eventType: "step_started",
3067
3147
  payload: {
3068
- commandBatchId,
3069
- commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3148
+ previousStepOrder: prevStep,
3149
+ stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3070
3150
  },
3071
3151
  currentStepOrder: payload.stepIndex
3072
3152
  });
3073
3153
  }
3074
- if (typeof payload.stepIndex === "number") {
3075
- const prevStep = stepIndexRef.current;
3076
- stepIndexRef.current = payload.stepIndex;
3077
- setCurrentStepIndex(payload.stepIndex);
3078
- if (payload.stepIndex !== prevStep) {
3079
- const tour = tourRef.current;
3080
- const total = tour?.steps?.length ?? 0;
3081
- if (tour && total > 0) {
3082
- onStepChangeRef.current?.(payload.stepIndex, total, tour);
3083
- }
3084
- }
3085
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3086
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3087
- stepOrder: payload.stepIndex,
3088
- eventType: "step_started",
3089
- payload: {
3090
- previousStepOrder: prevStep,
3091
- stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3092
- },
3093
- currentStepOrder: payload.stepIndex
3094
- });
3095
- }
3096
- }
3097
- if (!payload.commands || !Array.isArray(payload.commands)) {
3098
- console.warn("[TourClient] Payload commands is not an array:", payload);
3099
- commandInFlightRef.current = false;
3100
- emitIfOpen("tour:action_result", {
3101
- success: false,
3102
- reason: "invalid_commands",
3103
- commandBatchId,
3104
- runId: runIdRef.current,
3105
- turnId: turnIdRef.current
3106
- });
3107
- clearCommandBatchId();
3108
- return;
3154
+ }
3155
+ if (!payload.commands || !Array.isArray(payload.commands)) {
3156
+ console.warn("[TourClient] Payload commands is not an array:", payload);
3157
+ commandInFlightRef.current = false;
3158
+ emitIfOpen("tour:action_result", {
3159
+ success: false,
3160
+ reason: "invalid_commands",
3161
+ commandBatchId,
3162
+ runId: runIdRef.current,
3163
+ turnId: turnIdRef.current
3164
+ });
3165
+ clearCommandBatchId();
3166
+ return;
3167
+ }
3168
+ let shouldWait = false;
3169
+ const results = [];
3170
+ let batchPreferredWaitTarget = null;
3171
+ const assertNotInterrupted = () => {
3172
+ if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3173
+ const error = new Error("interrupted");
3174
+ error.code = "INTERRUPTED";
3175
+ throw error;
3109
3176
  }
3110
- let shouldWait = false;
3111
- const results = [];
3112
- let batchPreferredWaitTarget = null;
3113
- const assertNotInterrupted = () => {
3114
- if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3115
- const error = new Error("interrupted");
3116
- error.code = "INTERRUPTED";
3117
- throw error;
3118
- }
3119
- };
3120
- const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3121
- const fallbackHints = fallbackStep?.element ?? null;
3122
- return retryLookup({
3123
- timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3124
- pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3125
- onRetry: () => {
3126
- assertNotInterrupted();
3127
- },
3128
- resolve: async () => {
3129
- let targetEl = null;
3130
- if (params.uid) {
3131
- const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
3132
- targetEl = getElementByUid(params.uid);
3133
- }
3134
- if (!targetEl) {
3135
- targetEl = resolveElementFromHints({
3136
- fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3137
- testId: params.testId ?? fallbackHints?.testId,
3138
- textContaining: params.textContaining ?? fallbackHints?.textContaining
3139
- }, fallbackStep, tagStore);
3140
- }
3141
- return targetEl;
3142
- }
3143
- });
3144
- };
3145
- const executeOne = async (action) => {
3146
- assertNotInterrupted();
3147
- console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3148
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3149
- const executeTimeline = async (params = {}) => {
3150
- const segments = Array.isArray(params.segments) ? params.segments : [];
3151
- for (let index = 0; index < segments.length; index += 1) {
3152
- assertNotInterrupted();
3153
- const segment = segments[index];
3154
- const segmentText = (segment?.text ?? "").trim();
3155
- const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3156
- const events = Array.isArray(segment?.events) ? segment.events : [];
3157
- if (segmentDelayMs > 0) {
3158
- await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3159
- }
3160
- if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3161
- const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3162
- if (!gateTarget) {
3163
- throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3164
- }
3165
- await waitForUserAction(gateTarget, segment.gate.event);
3166
- }
3167
- if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3168
- showCaption(segmentText);
3169
- }
3170
- const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3171
- const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3172
- prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3173
- onNearEnd: nextSegmentText ? () => {
3174
- void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3175
- } : void 0
3176
- }) : Promise.resolve();
3177
- const eventsPromise = (async () => {
3178
- for (const event of events) {
3179
- assertNotInterrupted();
3180
- const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3181
- if (delayMs > 0) {
3182
- await new Promise((resolve) => setTimeout(resolve, delayMs));
3183
- }
3184
- if (event?.action) {
3185
- await executeOne(event.action);
3186
- }
3187
- }
3188
- })();
3189
- await Promise.all([speechPromise, eventsPromise]);
3190
- }
3191
- if (params.removeHighlightAtEnd !== false) {
3192
- removeHighlight();
3193
- }
3194
- if (showCaptionsRef.current && reviewModeRef.current) {
3195
- removeCaption();
3196
- }
3197
- return !!params.waitForInput;
3198
- };
3199
- if (action.type === "speak") {
3200
- const text = action.params?.text ?? "";
3201
- if (!text.trim()) {
3202
- return { result: null };
3177
+ };
3178
+ const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3179
+ const fallbackHints = fallbackStep?.element ?? null;
3180
+ return retryLookup({
3181
+ timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3182
+ pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3183
+ onRetry: () => {
3184
+ assertNotInterrupted();
3185
+ },
3186
+ resolve: async () => {
3187
+ let targetEl = null;
3188
+ if (params.uid) {
3189
+ const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
3190
+ targetEl = getElementByUid(params.uid);
3203
3191
  }
3204
- if (showCaptionsRef.current && reviewModeRef.current) {
3205
- showCaption(text);
3192
+ if (!targetEl) {
3193
+ targetEl = resolveElementFromHints({
3194
+ fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3195
+ testId: params.testId ?? fallbackHints?.testId,
3196
+ textContaining: params.textContaining ?? fallbackHints?.textContaining
3197
+ }, fallbackStep, tagStore);
3206
3198
  }
3207
- const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3208
- // Schedule narration immediately so the client keeps its speculative,
3209
- // low-latency behavior, but defer batch completion until playback settles.
3210
- waitForCompletion: false,
3211
- interrupt: action.params?.interrupt
3212
- });
3213
- return { result: text, settlePromise };
3199
+ return targetEl;
3214
3200
  }
3215
- if (action.type === "play_timeline") {
3216
- const timelineShouldWait = await executeTimeline(action.params);
3217
- if (timelineShouldWait) shouldWait = true;
3218
- return { result: "timeline_executed" };
3219
- }
3220
- if (action.type === "highlight_element") {
3221
- const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3222
- if (resolvedEl) {
3223
- if (isEditableWaitTarget(resolvedEl)) {
3224
- batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3201
+ });
3202
+ };
3203
+ const executeOne = async (action) => {
3204
+ assertNotInterrupted();
3205
+ console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3206
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3207
+ const executeTimeline = async (params = {}) => {
3208
+ const segments = Array.isArray(params.segments) ? params.segments : [];
3209
+ for (let index = 0; index < segments.length; index += 1) {
3210
+ assertNotInterrupted();
3211
+ const segment = segments[index];
3212
+ const segmentText = (segment?.text ?? "").trim();
3213
+ const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3214
+ const events = Array.isArray(segment?.events) ? segment.events : [];
3215
+ if (segmentDelayMs > 0) {
3216
+ await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3217
+ }
3218
+ if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3219
+ const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3220
+ if (!gateTarget) {
3221
+ throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3225
3222
  }
3226
- showHighlight(resolvedEl, action.params?.label || action.label);
3227
- resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3228
- return { result: "highlighted" };
3223
+ await waitForUserAction(gateTarget, segment.gate.event);
3229
3224
  }
3230
- if (!action.params?.optional) {
3231
- throw new Error(
3232
- `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3233
- );
3225
+ if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3226
+ showCaption(segmentText);
3234
3227
  }
3235
- return { result: "highlight_optional_miss" };
3228
+ const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3229
+ const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3230
+ prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3231
+ onNearEnd: nextSegmentText ? () => {
3232
+ void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3233
+ } : void 0
3234
+ }) : Promise.resolve();
3235
+ const eventsPromise = (async () => {
3236
+ for (const event of events) {
3237
+ assertNotInterrupted();
3238
+ const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3239
+ if (delayMs > 0) {
3240
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
3241
+ }
3242
+ if (event?.action) {
3243
+ await executeOne(event.action);
3244
+ }
3245
+ }
3246
+ })();
3247
+ await Promise.all([speechPromise, eventsPromise]);
3236
3248
  }
3237
- if (action.type === "remove_highlight") {
3249
+ if (params.removeHighlightAtEnd !== false) {
3238
3250
  removeHighlight();
3239
- return { result: "highlight_removed" };
3240
3251
  }
3241
- if (action.type === "click_element") {
3242
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3243
- if (!targetEl) {
3244
- if (action.params?.optional) return { result: "click_optional_miss" };
3245
- throw new Error(
3246
- `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3247
- );
3248
- }
3249
- removeHighlight();
3250
- await performInteractiveClick(targetEl);
3251
- const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
3252
- await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3253
- return { result: "clicked" };
3252
+ if (showCaptionsRef.current && reviewModeRef.current) {
3253
+ removeCaption();
3254
3254
  }
3255
- if (action.type === "fill_input") {
3256
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3257
- if (!targetEl) {
3258
- throw new Error(
3259
- `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3260
- );
3261
- }
3262
- const value = typeof action.params?.value === "string" ? action.params.value : "";
3263
- batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3264
- await performInteractiveFill(targetEl, value);
3265
- return { result: value };
3255
+ return !!params.waitForInput;
3256
+ };
3257
+ if (action.type === "speak") {
3258
+ const text = action.params?.text ?? "";
3259
+ if (!text.trim()) {
3260
+ return { result: null };
3266
3261
  }
3267
- if (action.type === "take_screenshot") {
3268
- const html2canvasModule = await import("html2canvas");
3269
- const html2canvas2 = html2canvasModule.default;
3270
- const canvas = await html2canvas2(document.body, {
3271
- useCORS: true,
3272
- allowTaint: true,
3273
- scale: Math.min(window.devicePixelRatio, 2),
3274
- width: window.innerWidth,
3275
- height: window.innerHeight,
3276
- x: window.scrollX,
3277
- y: window.scrollY,
3278
- logging: false
3279
- });
3280
- return { result: canvas.toDataURL("image/png") };
3262
+ if (showCaptionsRef.current && reviewModeRef.current) {
3263
+ showCaption(text);
3281
3264
  }
3282
- if (action.type === "navigate_to_url") {
3283
- const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3284
- if (!nextUrl) {
3285
- throw new Error("navigate_to_url missing url");
3265
+ const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3266
+ // Schedule narration immediately so the client keeps its speculative,
3267
+ // low-latency behavior, but defer batch completion until playback settles.
3268
+ waitForCompletion: false,
3269
+ interrupt: action.params?.interrupt
3270
+ });
3271
+ return { result: text, settlePromise };
3272
+ }
3273
+ if (action.type === "play_timeline") {
3274
+ const timelineShouldWait = await executeTimeline(action.params);
3275
+ if (timelineShouldWait) shouldWait = true;
3276
+ return { result: "timeline_executed" };
3277
+ }
3278
+ if (action.type === "highlight_element") {
3279
+ const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3280
+ if (resolvedEl) {
3281
+ if (isEditableWaitTarget(resolvedEl)) {
3282
+ batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3286
3283
  }
3287
- await navigateToTourUrl(nextUrl);
3288
- const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
3289
- await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3290
- return { result: nextUrl };
3284
+ showHighlight(resolvedEl, action.params?.label || action.label);
3285
+ resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3286
+ return { result: "highlighted" };
3291
3287
  }
3292
- if (action.type === "execute_agent_action") {
3293
- const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3294
- if (!agentSocketId) {
3295
- throw new Error("No socketId available for execute_agent_action");
3296
- }
3297
- const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3298
- const res = await fetch(url, {
3299
- method: "POST",
3300
- headers: { "Content-Type": "application/json" },
3301
- body: JSON.stringify({
3302
- command: action.params?.command,
3303
- socketId: agentSocketId
3304
- })
3305
- });
3306
- if (!res.ok) {
3307
- throw new Error(`execute_agent_action failed: ${await res.text()}`);
3308
- }
3309
- if (action.params?.wait !== false) {
3310
- await res.json();
3311
- }
3312
- const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3313
- await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
3314
- await syncAOM();
3315
- return { result: action.params?.command ?? "executed" };
3288
+ if (!action.params?.optional) {
3289
+ throw new Error(
3290
+ `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3291
+ );
3316
3292
  }
3317
- if (action.type === "wait_for_user_action") {
3318
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3319
- if (!targetEl) {
3320
- throw new Error("wait_for_user_action target not found");
3321
- }
3322
- const eventName = action.params?.event ?? "click";
3323
- await waitForUserAction(targetEl, eventName);
3324
- return { result: `waited_for_${eventName}` };
3293
+ return { result: "highlight_optional_miss" };
3294
+ }
3295
+ if (action.type === "remove_highlight") {
3296
+ removeHighlight();
3297
+ return { result: "highlight_removed" };
3298
+ }
3299
+ if (action.type === "click_element") {
3300
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3301
+ if (!targetEl) {
3302
+ if (action.params?.optional) return { result: "click_optional_miss" };
3303
+ throw new Error(
3304
+ `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3305
+ );
3325
3306
  }
3326
- if (action.type === "wait_for_input") {
3327
- shouldWait = true;
3328
- return { result: "waiting_for_input" };
3307
+ removeHighlight();
3308
+ await performInteractiveClick(targetEl);
3309
+ const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
3310
+ await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3311
+ return { result: "clicked" };
3312
+ }
3313
+ if (action.type === "fill_input") {
3314
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3315
+ if (!targetEl) {
3316
+ throw new Error(
3317
+ `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3318
+ );
3329
3319
  }
3330
- if (action.type === "end_tour") {
3331
- handleTourEnd();
3332
- return { result: "ended" };
3320
+ const value = typeof action.params?.value === "string" ? action.params.value : "";
3321
+ batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3322
+ await performInteractiveFill(targetEl, value);
3323
+ return { result: value };
3324
+ }
3325
+ if (action.type === "take_screenshot") {
3326
+ const html2canvasModule = await import("html2canvas");
3327
+ const html2canvas2 = html2canvasModule.default;
3328
+ const canvas = await html2canvas2(document.body, {
3329
+ useCORS: true,
3330
+ allowTaint: true,
3331
+ scale: Math.min(window.devicePixelRatio, 2),
3332
+ width: window.innerWidth,
3333
+ height: window.innerHeight,
3334
+ x: window.scrollX,
3335
+ y: window.scrollY,
3336
+ logging: false
3337
+ });
3338
+ return { result: canvas.toDataURL("image/png") };
3339
+ }
3340
+ if (action.type === "navigate_to_url") {
3341
+ const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3342
+ if (!nextUrl) {
3343
+ throw new Error("navigate_to_url missing url");
3333
3344
  }
3334
- console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3335
- return { result: "unknown_action_skipped" };
3336
- };
3337
- try {
3338
- const resultsBuffer = new Array(payload.commands.length);
3339
- const pendingUIActions = [];
3340
- for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3341
- const command = payload.commands[commandIndex];
3342
- assertNotInterrupted();
3343
- const isTerminal = isTerminalAction(command);
3344
- if (isTerminal) {
3345
- await Promise.all(pendingUIActions);
3346
- pendingUIActions.length = 0;
3347
- }
3348
- const executionPromise = (async () => {
3349
- const execution = await executeOne(command);
3350
- await execution.settlePromise;
3351
- resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3352
- })();
3353
- if (isTerminal) {
3354
- await executionPromise;
3355
- } else {
3356
- pendingUIActions.push(executionPromise);
3357
- }
3345
+ await navigateToTourUrl(nextUrl);
3346
+ const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
3347
+ await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3348
+ return { result: nextUrl };
3349
+ }
3350
+ if (action.type === "execute_agent_action") {
3351
+ const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3352
+ if (!agentSocketId) {
3353
+ throw new Error("No socketId available for execute_agent_action");
3358
3354
  }
3359
- await Promise.all(pendingUIActions);
3360
- resultsBuffer.forEach((res) => {
3361
- if (res) results.push(res);
3355
+ const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3356
+ const res = await fetch(url, {
3357
+ method: "POST",
3358
+ headers: { "Content-Type": "application/json" },
3359
+ body: JSON.stringify({
3360
+ command: action.params?.command,
3361
+ socketId: agentSocketId
3362
+ })
3362
3363
  });
3363
- await syncAOM();
3364
- } catch (err) {
3365
- commandInFlightRef.current = false;
3366
- const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3367
- if (interrupted) {
3368
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3369
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3370
- stepOrder: stepIndexRef.current,
3371
- eventType: "command_batch_interrupted",
3372
- payload: {
3373
- commandBatchId,
3374
- partialResults: results
3375
- },
3376
- currentStepOrder: stepIndexRef.current
3377
- });
3378
- }
3379
- emitIfOpen("tour:action_result", {
3380
- success: true,
3381
- interrupted: true,
3382
- results,
3383
- commandBatchId,
3384
- runId: runIdRef.current,
3385
- turnId: turnIdRef.current
3386
- });
3387
- clearCommandBatchId();
3388
- return;
3364
+ if (!res.ok) {
3365
+ throw new Error(`execute_agent_action failed: ${await res.text()}`);
3389
3366
  }
3390
- console.error("[TourClient] Command batch execution failed:", err);
3391
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3392
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3393
- stepOrder: stepIndexRef.current,
3394
- eventType: "command_batch_failed",
3395
- payload: {
3396
- commandBatchId,
3397
- error: String(err),
3398
- partialResults: results
3399
- },
3400
- status: "active",
3401
- currentStepOrder: stepIndexRef.current
3402
- });
3367
+ if (action.params?.wait !== false) {
3368
+ await res.json();
3403
3369
  }
3404
- emitIfOpen("tour:action_result", {
3405
- success: false,
3406
- reason: "execution_error",
3407
- error: String(err),
3408
- results,
3409
- commandBatchId,
3410
- runId: runIdRef.current,
3411
- turnId: turnIdRef.current
3412
- });
3413
- clearCommandBatchId();
3414
- return;
3370
+ const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3371
+ await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
3372
+ await syncAOM();
3373
+ return { result: action.params?.command ?? "executed" };
3415
3374
  }
3416
- commandInFlightRef.current = false;
3417
- if (shouldWait && !skipRequestedRef.current) {
3418
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3419
- const waitCondition = currentStep?.onboarding?.waitCondition;
3420
- const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3421
- const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3422
- const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3423
- let manualWaitPromise = null;
3424
- let manualWaitKind = null;
3425
- const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3426
- const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3427
- runCleanup(pendingManualWaitCleanupRef.current);
3428
- pendingManualWaitCleanupRef.current = null;
3429
- if (waitTargetHints) {
3430
- let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3431
- if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3432
- manualWaitTarget = preferredWaitTarget;
3433
- console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3434
- }
3435
- if (manualWaitTarget) {
3436
- const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3437
- manualWaitPromise = manualWait.promise;
3438
- manualWaitKind = manualWait.kind;
3439
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3440
- }
3375
+ if (action.type === "wait_for_user_action") {
3376
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3377
+ if (!targetEl) {
3378
+ throw new Error("wait_for_user_action target not found");
3441
3379
  }
3442
- if (!manualWaitPromise && preferredWaitTarget) {
3443
- const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3444
- manualWaitPromise = manualWait.promise;
3445
- manualWaitKind = manualWait.kind;
3446
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3447
- console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3380
+ const eventName = action.params?.event ?? "click";
3381
+ await waitForUserAction(targetEl, eventName);
3382
+ return { result: `waited_for_${eventName}` };
3383
+ }
3384
+ if (action.type === "wait_for_input") {
3385
+ shouldWait = true;
3386
+ return { result: "waiting_for_input" };
3387
+ }
3388
+ if (action.type === "end_tour") {
3389
+ handleTourEnd();
3390
+ return { result: "ended" };
3391
+ }
3392
+ console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3393
+ return { result: "unknown_action_skipped" };
3394
+ };
3395
+ try {
3396
+ const resultsBuffer = new Array(payload.commands.length);
3397
+ const pendingUIActions = [];
3398
+ for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3399
+ const command = payload.commands[commandIndex];
3400
+ assertNotInterrupted();
3401
+ const isTerminal = isTerminalAction(command);
3402
+ if (isTerminal) {
3403
+ await Promise.all(pendingUIActions);
3404
+ pendingUIActions.length = 0;
3448
3405
  }
3449
- if (!manualWaitPromise && inputLikeWait) {
3450
- const firstInput = document.querySelector(
3451
- 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3452
- );
3453
- if (firstInput) {
3454
- const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3455
- manualWaitPromise = manualWait.promise;
3456
- manualWaitKind = manualWait.kind;
3457
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3458
- console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3459
- }
3406
+ const executionPromise = (async () => {
3407
+ const execution = await executeOne(command);
3408
+ await execution.settlePromise;
3409
+ resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3410
+ })();
3411
+ if (isTerminal) {
3412
+ await executionPromise;
3413
+ } else {
3414
+ pendingUIActions.push(executionPromise);
3460
3415
  }
3461
- setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3416
+ }
3417
+ await Promise.all(pendingUIActions);
3418
+ resultsBuffer.forEach((res) => {
3419
+ if (res) results.push(res);
3420
+ });
3421
+ await syncAOM();
3422
+ } catch (err) {
3423
+ commandInFlightRef.current = false;
3424
+ const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3425
+ if (interrupted) {
3462
3426
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3463
3427
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3464
3428
  stepOrder: stepIndexRef.current,
3465
- eventType: "waiting_for_input",
3429
+ eventType: "command_batch_interrupted",
3466
3430
  payload: {
3467
3431
  commandBatchId,
3468
- results
3432
+ partialResults: results
3469
3433
  },
3470
3434
  currentStepOrder: stepIndexRef.current
3471
3435
  });
3472
3436
  }
3473
3437
  emitIfOpen("tour:action_result", {
3474
3438
  success: true,
3475
- waitingForInput: true,
3439
+ interrupted: true,
3476
3440
  results,
3477
3441
  commandBatchId,
3478
3442
  runId: runIdRef.current,
3479
3443
  turnId: turnIdRef.current
3480
3444
  });
3481
3445
  clearCommandBatchId();
3482
- const voiceOrTextWaitPromise = new Promise((resolve) => {
3483
- if (pendingInputBufRef.current) {
3484
- const flushed = pendingInputBufRef.current;
3485
- pendingInputBufRef.current = null;
3486
- resolve(flushed);
3487
- return;
3488
- }
3489
- voiceInputResolveRef.current = (text) => {
3490
- voiceInputResolveRef.current = null;
3491
- resolve(text);
3492
- };
3493
- });
3494
- Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3495
- runCleanup(pendingManualWaitCleanupRef.current);
3496
- pendingManualWaitCleanupRef.current = null;
3497
- voiceInputResolveRef.current = null;
3498
- setPlaybackState("executing");
3499
- const transcript = userText.trim();
3500
- if (!transcript) {
3501
- return;
3502
- }
3503
- const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3504
- await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
3505
- await syncAOM();
3506
- emitIfOpen("tour:user_input", {
3507
- transcript,
3508
- runId: runIdRef.current,
3509
- turnId: turnIdRef.current
3510
- });
3511
- });
3512
3446
  return;
3513
3447
  }
3448
+ console.error("[TourClient] Command batch execution failed:", err);
3514
3449
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3515
3450
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3516
3451
  stepOrder: stepIndexRef.current,
3517
- eventType: "command_batch_completed",
3452
+ eventType: "command_batch_failed",
3518
3453
  payload: {
3519
3454
  commandBatchId,
3520
- results
3455
+ error: String(err),
3456
+ partialResults: results
3521
3457
  },
3458
+ status: "active",
3522
3459
  currentStepOrder: stepIndexRef.current
3523
3460
  });
3524
3461
  }
3525
3462
  emitIfOpen("tour:action_result", {
3526
- success: true,
3463
+ success: false,
3464
+ reason: "execution_error",
3465
+ error: String(err),
3527
3466
  results,
3528
3467
  commandBatchId,
3529
3468
  runId: runIdRef.current,
3530
3469
  turnId: turnIdRef.current
3531
3470
  });
3532
3471
  clearCommandBatchId();
3533
- });
3534
- socket.on("tour:start", async (tourData) => {
3535
- if (isActiveRef.current) return;
3536
- runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3537
- const tour = tourData.tourContext ?? tourRef.current;
3538
- const expType = experienceTypeRef.current;
3539
- if (tour?.type && tour.type !== expType) {
3540
- console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3541
- return;
3472
+ return;
3473
+ }
3474
+ commandInFlightRef.current = false;
3475
+ if (shouldWait && !skipRequestedRef.current) {
3476
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3477
+ const waitCondition = currentStep?.onboarding?.waitCondition;
3478
+ const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3479
+ const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3480
+ const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3481
+ let manualWaitPromise = null;
3482
+ let manualWaitKind = null;
3483
+ const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3484
+ const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3485
+ runCleanup(pendingManualWaitCleanupRef.current);
3486
+ pendingManualWaitCleanupRef.current = null;
3487
+ if (waitTargetHints) {
3488
+ let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3489
+ if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3490
+ manualWaitTarget = preferredWaitTarget;
3491
+ console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3492
+ }
3493
+ if (manualWaitTarget) {
3494
+ const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3495
+ manualWaitPromise = manualWait.promise;
3496
+ manualWaitKind = manualWait.kind;
3497
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3498
+ }
3499
+ }
3500
+ if (!manualWaitPromise && preferredWaitTarget) {
3501
+ const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3502
+ manualWaitPromise = manualWait.promise;
3503
+ manualWaitKind = manualWait.kind;
3504
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3505
+ console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3542
3506
  }
3543
- skipRequestedRef.current = false;
3544
- const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3545
- isActiveRef.current = true;
3546
- setIsActive(true);
3547
- setActiveTour(tour ?? null);
3548
- tourRef.current = tour ?? null;
3549
- setTotalSteps(total);
3550
- stepIndexRef.current = 0;
3551
- setCurrentStepIndex(0);
3552
- setPlaybackState("intro");
3553
- if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3554
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3555
- stepOrder: 0,
3556
- eventType: "tour_started",
3507
+ if (!manualWaitPromise && inputLikeWait) {
3508
+ const firstInput = document.querySelector(
3509
+ 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3510
+ );
3511
+ if (firstInput) {
3512
+ const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3513
+ manualWaitPromise = manualWait.promise;
3514
+ manualWaitKind = manualWait.kind;
3515
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3516
+ console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3517
+ }
3518
+ }
3519
+ setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3520
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3521
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3522
+ stepOrder: stepIndexRef.current,
3523
+ eventType: "waiting_for_input",
3557
3524
  payload: {
3558
- totalSteps: total,
3559
- source: "sdk_test_preview"
3525
+ commandBatchId,
3526
+ results
3560
3527
  },
3561
- currentStepOrder: 0
3528
+ currentStepOrder: stepIndexRef.current
3562
3529
  });
3563
3530
  }
3564
- try {
3565
- const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3566
- const aom = generateMinifiedAOM2();
3567
- if (socketRef.current === socket && socket.connected) {
3568
- socket.emit("tour:sync_dom", {
3569
- url: window.location.pathname + window.location.search + window.location.hash,
3570
- aom: aom.nodes,
3571
- domSummary: captureDomSummary()
3572
- });
3531
+ emitIfOpen("tour:action_result", {
3532
+ success: true,
3533
+ waitingForInput: true,
3534
+ results,
3535
+ commandBatchId,
3536
+ runId: runIdRef.current,
3537
+ turnId: turnIdRef.current
3538
+ });
3539
+ clearCommandBatchId();
3540
+ const voiceOrTextWaitPromise = new Promise((resolve) => {
3541
+ if (pendingInputBufRef.current) {
3542
+ const flushed = pendingInputBufRef.current;
3543
+ pendingInputBufRef.current = null;
3544
+ resolve(flushed);
3545
+ return;
3573
3546
  }
3574
- } catch (e) {
3575
- console.warn("[TourClient] Initial DOM sync failed:", e);
3576
- }
3547
+ voiceInputResolveRef.current = (text) => {
3548
+ voiceInputResolveRef.current = null;
3549
+ resolve(text);
3550
+ };
3551
+ });
3552
+ Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3553
+ runCleanup(pendingManualWaitCleanupRef.current);
3554
+ pendingManualWaitCleanupRef.current = null;
3555
+ voiceInputResolveRef.current = null;
3556
+ setPlaybackState("executing");
3557
+ const transcript = userText.trim();
3558
+ if (!transcript) {
3559
+ return;
3560
+ }
3561
+ const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
3562
+ await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
3563
+ await syncAOM();
3564
+ emitIfOpen("tour:user_input", {
3565
+ transcript,
3566
+ runId: runIdRef.current,
3567
+ turnId: turnIdRef.current
3568
+ });
3569
+ });
3570
+ return;
3571
+ }
3572
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3573
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3574
+ stepOrder: stepIndexRef.current,
3575
+ eventType: "command_batch_completed",
3576
+ payload: {
3577
+ commandBatchId,
3578
+ results
3579
+ },
3580
+ currentStepOrder: stepIndexRef.current
3581
+ });
3582
+ }
3583
+ emitIfOpen("tour:action_result", {
3584
+ success: true,
3585
+ results,
3586
+ commandBatchId,
3587
+ runId: runIdRef.current,
3588
+ turnId: turnIdRef.current
3577
3589
  });
3578
- socket.on("tour:update", (payload) => {
3579
- const updatedTour = payload?.tourContext;
3580
- if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3581
- return;
3590
+ clearCommandBatchId();
3591
+ };
3592
+ const handleTourStart = async (tourData) => {
3593
+ if (isActiveRef.current) return;
3594
+ runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3595
+ const tour = tourData.tourContext ?? tourRef.current;
3596
+ const expType = experienceTypeRef.current;
3597
+ if (tour?.type && tour.type !== expType) {
3598
+ console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3599
+ return;
3600
+ }
3601
+ skipRequestedRef.current = false;
3602
+ const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3603
+ isActiveRef.current = true;
3604
+ setIsActive(true);
3605
+ setActiveTour(tour ?? null);
3606
+ tourRef.current = tour ?? null;
3607
+ setTotalSteps(total);
3608
+ stepIndexRef.current = 0;
3609
+ setCurrentStepIndex(0);
3610
+ setPlaybackState("intro");
3611
+ if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3612
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3613
+ stepOrder: 0,
3614
+ eventType: "tour_started",
3615
+ payload: {
3616
+ totalSteps: total,
3617
+ source: "sdk_test_preview"
3618
+ },
3619
+ currentStepOrder: 0
3620
+ });
3621
+ }
3622
+ try {
3623
+ const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3624
+ const aom = generateMinifiedAOM2();
3625
+ if (socketRef.current === socket) {
3626
+ emitSocketEvent(socket, "tour:sync_dom", {
3627
+ url: window.location.pathname + window.location.search + window.location.hash,
3628
+ aom: aom.nodes,
3629
+ domSummary: captureDomSummary()
3630
+ });
3582
3631
  }
3583
- tourRef.current = updatedTour;
3584
- setActiveTour(updatedTour);
3585
- const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3586
- setTotalSteps(nextTotal);
3587
- if (typeof payload.currentStepIndex === "number") {
3588
- const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3589
- stepIndexRef.current = clampedStepIndex;
3590
- setCurrentStepIndex(clampedStepIndex);
3591
- if (nextTotal > 0) {
3592
- onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3593
- }
3632
+ } catch (e) {
3633
+ console.warn("[TourClient] Initial DOM sync failed:", e);
3634
+ }
3635
+ };
3636
+ const handleTourUpdate = (payload) => {
3637
+ const updatedTour = payload?.tourContext;
3638
+ if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3639
+ return;
3640
+ }
3641
+ tourRef.current = updatedTour;
3642
+ setActiveTour(updatedTour);
3643
+ const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3644
+ setTotalSteps(nextTotal);
3645
+ if (typeof payload.currentStepIndex === "number") {
3646
+ const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3647
+ stepIndexRef.current = clampedStepIndex;
3648
+ setCurrentStepIndex(clampedStepIndex);
3649
+ if (nextTotal > 0) {
3650
+ onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3594
3651
  }
3595
- });
3596
- socket.on("tour:end", () => {
3597
- setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3598
- handleTourEnd();
3599
- });
3600
- socket.on("tour:debug_log", (entry) => {
3601
- const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3602
- if (isDev) {
3603
- console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3604
- if (typeof window !== "undefined") {
3605
- window.dispatchEvent(new CustomEvent("modelnex-debug", {
3606
- detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3607
- }));
3608
- }
3652
+ }
3653
+ };
3654
+ const handleTourEndEvent = () => {
3655
+ setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3656
+ handleTourEnd();
3657
+ };
3658
+ const handleDebugLog = (entry) => {
3659
+ const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3660
+ if (isDev) {
3661
+ console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3662
+ if (typeof window !== "undefined") {
3663
+ window.dispatchEvent(new CustomEvent("modelnex-debug", {
3664
+ detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3665
+ }));
3609
3666
  }
3610
- });
3611
- console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3612
- });
3613
- return () => {
3614
- cancelled = true;
3615
- const toClose = createdSocket ?? socketRef.current;
3616
- if (toClose) {
3617
- toClose.disconnect();
3618
3667
  }
3619
- createdSocket = null;
3668
+ };
3669
+ socket.on("connect", handleConnect);
3670
+ socket.on("tour:server_state", handleServerState);
3671
+ socket.on("tour:command_cancel", handleCommandCancel);
3672
+ socket.on("tour:command", handleCommand);
3673
+ socket.on("tour:start", handleTourStart);
3674
+ socket.on("tour:update", handleTourUpdate);
3675
+ socket.on("tour:end", handleTourEndEvent);
3676
+ socket.on("tour:debug_log", handleDebugLog);
3677
+ console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3678
+ return () => {
3679
+ socket.off("connect", handleConnect);
3680
+ socket.off("tour:server_state", handleServerState);
3681
+ socket.off("tour:command_cancel", handleCommandCancel);
3682
+ socket.off("tour:command", handleCommand);
3683
+ socket.off("tour:start", handleTourStart);
3684
+ socket.off("tour:update", handleTourUpdate);
3685
+ socket.off("tour:end", handleTourEndEvent);
3686
+ socket.off("tour:debug_log", handleDebugLog);
3687
+ const toClose = socket;
3620
3688
  socketRef.current = null;
3621
3689
  setServerState(null);
3622
3690
  runIdRef.current = null;
3623
3691
  turnIdRef.current = null;
3692
+ tourSocketPool.release(serverUrl, toClose);
3624
3693
  };
3625
- }, [serverUrl, websiteId, disabled]);
3694
+ }, [serverUrl, disabled]);
3626
3695
  useEffect11(() => {
3627
3696
  if (disabled) return;
3628
3697
  const s = socketRef.current;
3629
3698
  const profile = userProfile;
3630
- if (!s?.connected || !websiteId || !profile?.userId) return;
3631
- s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3699
+ if (!websiteId || !profile?.userId) return;
3700
+ const timer = setTimeout(() => {
3701
+ emitSocketEvent(s, "tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3702
+ }, 150);
3703
+ return () => clearTimeout(timer);
3632
3704
  }, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
3633
3705
  useEffect11(() => {
3634
3706
  if (!showCaptions || !isReviewMode) {
@@ -3636,8 +3708,8 @@ function useTourPlayback({
3636
3708
  }
3637
3709
  }, [showCaptions, isReviewMode]);
3638
3710
  useEffect11(() => {
3639
- if (!socketRef.current?.connected || !isActiveRef.current) return;
3640
- socketRef.current.emit("tour:client_state", {
3711
+ if (!isActiveRef.current) return;
3712
+ emitSocketEvent(socketRef.current, "tour:client_state", {
3641
3713
  runId: runIdRef.current,
3642
3714
  turnId: turnIdRef.current,
3643
3715
  commandBatchId: activeCommandBatchIdRef.current,
@@ -3650,17 +3722,17 @@ function useTourPlayback({
3650
3722
  });
3651
3723
  }, [isActive, playbackState, voice.isListening, voice.isSpeaking]);
3652
3724
  const syncAOM = useCallback7(async () => {
3653
- if (!socketRef.current?.connected || !isActiveRef.current) return;
3725
+ if (!isActiveRef.current) return;
3654
3726
  const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
3655
3727
  const aom = generateMinifiedAOM2();
3656
- socketRef.current.emit("tour:sync_dom", {
3728
+ emitSocketEvent(socketRef.current, "tour:sync_dom", {
3657
3729
  url: window.location.pathname + window.location.search + window.location.hash,
3658
3730
  aom: aom.nodes,
3659
3731
  domSummary: captureDomSummary()
3660
3732
  });
3661
3733
  }, []);
3662
3734
  const interruptExecution = useCallback7((transcript) => {
3663
- if (!socketRef.current?.connected || !isActiveRef.current) return false;
3735
+ if (!isSocketWritable(socketRef.current) || !isActiveRef.current) return false;
3664
3736
  if (!commandInFlightRef.current && !voice.isSpeaking) return false;
3665
3737
  interruptedForQuestionRef.current = true;
3666
3738
  activeExecutionTokenRef.current += 1;
@@ -3668,21 +3740,22 @@ function useTourPlayback({
3668
3740
  removeHighlight();
3669
3741
  removeCaption();
3670
3742
  voice.stopSpeaking();
3671
- socketRef.current.emit("tour:action_result", {
3743
+ if (emitSocketEvent(socketRef.current, "tour:action_result", {
3672
3744
  success: true,
3673
3745
  interrupted: true,
3674
3746
  transcript,
3675
3747
  commandBatchId: activeCommandBatchIdRef.current,
3676
3748
  runId: runIdRef.current,
3677
3749
  turnId: turnIdRef.current
3678
- });
3679
- activeCommandBatchIdRef.current = null;
3680
- socketRef.current.emit("tour:user_input", {
3681
- transcript,
3682
- interrupted: true,
3683
- runId: runIdRef.current,
3684
- turnId: turnIdRef.current
3685
- });
3750
+ })) {
3751
+ activeCommandBatchIdRef.current = null;
3752
+ emitSocketEvent(socketRef.current, "tour:user_input", {
3753
+ transcript,
3754
+ interrupted: true,
3755
+ runId: runIdRef.current,
3756
+ turnId: turnIdRef.current
3757
+ });
3758
+ }
3686
3759
  setPlaybackState("thinking");
3687
3760
  return true;
3688
3761
  }, [voice]);
@@ -3698,9 +3771,7 @@ function useTourPlayback({
3698
3771
  removeCaption();
3699
3772
  voice.stopSpeaking();
3700
3773
  voice.stopListening();
3701
- if (socketRef.current?.connected) {
3702
- socketRef.current.emit("tour:abort");
3703
- }
3774
+ emitSocketEvent(socketRef.current, "tour:abort");
3704
3775
  if (reviewModeRef.current && tourRef.current?.id && previewRunIdRef.current) {
3705
3776
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, tourRef.current.id, previewRunIdRef.current, websiteId, {
3706
3777
  stepOrder: stepIndexRef.current,
@@ -3772,7 +3843,7 @@ function useTourPlayback({
3772
3843
  await new Promise((r) => setTimeout(r, 200));
3773
3844
  retries++;
3774
3845
  }
3775
- if (!socketRef.current?.connected) {
3846
+ if (!isSocketWritable(socketRef.current)) {
3776
3847
  console.warn("[TourClient] Cannot run tour, socket not connected.");
3777
3848
  return;
3778
3849
  }
@@ -3800,7 +3871,7 @@ function useTourPlayback({
3800
3871
  previewRunIdRef.current = null;
3801
3872
  }
3802
3873
  tourRef.current = tour;
3803
- socketRef.current.emit("tour:request_start", {
3874
+ emitSocketEvent(socketRef.current, "tour:request_start", {
3804
3875
  tourId: tour.id,
3805
3876
  previewRunId: previewRunIdRef.current,
3806
3877
  tourContext: tour
@@ -3948,8 +4019,8 @@ function useTourPlayback({
3948
4019
  const revisionVersion = Number(response?.revision?.versionNumber);
3949
4020
  const appliedMessage = Number.isFinite(revisionVersion) && revisionVersion > 0 ? `Applied to the draft as version ${revisionVersion}.` : "Correction applied to the draft.";
3950
4021
  setReviewStatusMessage(apply ? appliedMessage : "Correction saved for review.");
3951
- if (apply && playbackState === "paused" && socketRef.current?.connected && isActiveRef.current) {
3952
- socketRef.current.emit("tour:resume");
4022
+ if (apply && playbackState === "paused" && isActiveRef.current) {
4023
+ emitSocketEvent(socketRef.current, "tour:resume");
3953
4024
  setPlaybackState("executing");
3954
4025
  }
3955
4026
  } catch (err) {
@@ -3973,14 +4044,14 @@ function useTourPlayback({
3973
4044
  stopTour();
3974
4045
  }, [stopTour]);
3975
4046
  const pauseTour = useCallback7(() => {
3976
- if (socketRef.current?.connected && isActiveRef.current) {
3977
- socketRef.current.emit("tour:pause");
4047
+ if (isSocketWritable(socketRef.current) && isActiveRef.current) {
4048
+ emitSocketEvent(socketRef.current, "tour:pause");
3978
4049
  setPlaybackState("paused");
3979
4050
  }
3980
4051
  }, []);
3981
4052
  const resumeTour = useCallback7(() => {
3982
- if (socketRef.current?.connected && isActiveRef.current) {
3983
- socketRef.current.emit("tour:resume");
4053
+ if (isSocketWritable(socketRef.current) && isActiveRef.current) {
4054
+ emitSocketEvent(socketRef.current, "tour:resume");
3984
4055
  setPlaybackState("executing");
3985
4056
  }
3986
4057
  }, []);
@@ -4002,9 +4073,9 @@ function useTourPlayback({
4002
4073
  if (voiceInputResolveRef.current) {
4003
4074
  console.log("[TourAgent] Resolving loop waiting_voice with text:", text);
4004
4075
  voiceInputResolveRef.current(text);
4005
- } else if (socketRef.current?.connected) {
4076
+ } else if (isSocketWritable(socketRef.current)) {
4006
4077
  console.log("[TourAgent] Forwarding ambient voice to server:", text);
4007
- socketRef.current.emit("tour:user_input", {
4078
+ emitSocketEvent(socketRef.current, "tour:user_input", {
4008
4079
  transcript: text,
4009
4080
  runId: runIdRef.current,
4010
4081
  turnId: turnIdRef.current
@@ -5252,7 +5323,7 @@ function useVoice(serverUrl) {
5252
5323
  (async () => {
5253
5324
  try {
5254
5325
  const ioModule = await import("socket.io-client");
5255
- const io2 = ioModule.default || ioModule;
5326
+ const io3 = ioModule.default || ioModule;
5256
5327
  const stream = await navigator.mediaDevices.getUserMedia({
5257
5328
  audio: {
5258
5329
  echoCancellation: true,
@@ -5274,7 +5345,7 @@ function useVoice(serverUrl) {
5274
5345
  audioBitsPerSecond: 128e3
5275
5346
  });
5276
5347
  mediaRecorderRef.current = recorder;
5277
- const socket = io2(serverUrl, {
5348
+ const socket = io3(serverUrl, {
5278
5349
  path: "/socket.io",
5279
5350
  transports: resolveSocketIoTransports(serverUrl, "websocket-first")
5280
5351
  });
@@ -10794,7 +10865,7 @@ var ModelNexProvider = ({
10794
10865
  socketId,
10795
10866
  devMode
10796
10867
  }),
10797
- [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
10868
+ [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
10798
10869
  );
10799
10870
  return React8.createElement(
10800
10871
  ModelNexContext.Provider,