@modelnex/sdk 0.5.14 → 0.5.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -2324,7 +2324,7 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
2324
2324
  }
2325
2325
 
2326
2326
  // src/constants.ts
2327
- var DEFAULT_MODELNEX_SERVER_URL = "https://modelnex-server-production.up.railway.app";
2327
+ var DEFAULT_MODELNEX_SERVER_URL = "https://api.modelnex.com";
2328
2328
 
2329
2329
  // src/hooks/useRunCommand.ts
2330
2330
  var import_react9 = require("react");
@@ -2626,6 +2626,7 @@ function isTourEligible(tour, userProfile) {
2626
2626
 
2627
2627
  // src/hooks/useTourPlayback.ts
2628
2628
  var import_react12 = require("react");
2629
+ var import_socket2 = require("socket.io-client");
2629
2630
 
2630
2631
  // src/utils/retryLookup.ts
2631
2632
  var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -3154,6 +3155,12 @@ function useTourPlayback({
3154
3155
  const [serverState, setServerState] = (0, import_react12.useState)(null);
3155
3156
  const ctx = (0, import_react12.useContext)(ModelNexContext);
3156
3157
  const devMode = ctx?.devMode;
3158
+ const devModeRef = (0, import_react12.useRef)(devMode);
3159
+ devModeRef.current = devMode;
3160
+ const userProfileRef = (0, import_react12.useRef)(userProfile);
3161
+ userProfileRef.current = userProfile;
3162
+ const experienceTypeRef = (0, import_react12.useRef)(experienceType);
3163
+ experienceTypeRef.current = experienceType;
3157
3164
  const tourRef = (0, import_react12.useRef)(null);
3158
3165
  const stepIndexRef = (0, import_react12.useRef)(0);
3159
3166
  const skipRequestedRef = (0, import_react12.useRef)(false);
@@ -3193,408 +3200,391 @@ function useTourPlayback({
3193
3200
  (0, import_react12.useEffect)(() => {
3194
3201
  if (disabled) return;
3195
3202
  if (typeof window === "undefined") return;
3196
- import("socket.io-client").then((ioModule) => {
3197
- const io2 = ioModule.default || ioModule;
3198
- const socket = io2(serverUrl, {
3199
- path: "/socket.io",
3200
- // standard
3201
- transports: resolveSocketIoTransports(serverUrl, "polling-first")
3202
- });
3203
- socketRef.current = socket;
3204
- socket.on("connect", () => {
3205
- console.log("[TourClient] Connected to tour agent server:", socket.id);
3206
- if (websiteId && userProfile) {
3207
- socket.emit("tour:init", { websiteId, userId: userProfile.userId, userType: userProfile.type });
3208
- }
3209
- });
3210
- socket.on("tour:server_state", (payload) => {
3211
- if (typeof payload?.runId === "number") {
3212
- runIdRef.current = payload.runId;
3213
- }
3214
- if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3215
- turnIdRef.current = payload.turnId ?? null;
3203
+ let createdSocket = null;
3204
+ const socket = (0, import_socket2.io)(serverUrl, {
3205
+ path: "/socket.io",
3206
+ transports: resolveSocketIoTransports(serverUrl, "polling-first"),
3207
+ autoConnect: true,
3208
+ reconnection: true,
3209
+ reconnectionAttempts: 10,
3210
+ reconnectionDelay: 1e3
3211
+ });
3212
+ createdSocket = socket;
3213
+ socketRef.current = socket;
3214
+ socket.on("connect", () => {
3215
+ console.log("[TourClient] Connected to tour agent server:", socket.id);
3216
+ const profile = userProfileRef.current;
3217
+ if (websiteId && profile?.userId && socket.connected) {
3218
+ socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3219
+ }
3220
+ });
3221
+ socket.on("tour:server_state", (payload) => {
3222
+ if (typeof payload?.runId === "number") {
3223
+ runIdRef.current = payload.runId;
3224
+ }
3225
+ if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3226
+ turnIdRef.current = payload.turnId ?? null;
3227
+ }
3228
+ setServerState(payload);
3229
+ });
3230
+ socket.on("tour:command_cancel", (payload) => {
3231
+ console.log("[TourClient] Received command_cancel:", payload);
3232
+ if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3233
+ activeCommandBatchIdRef.current = null;
3234
+ activeExecutionTokenRef.current++;
3235
+ commandInFlightRef.current = false;
3236
+ setPlaybackState("idle");
3237
+ if (typeof window !== "undefined" && window.speechSynthesis) {
3238
+ window.speechSynthesis.cancel();
3216
3239
  }
3217
- setServerState(payload);
3218
- });
3219
- socket.on("tour:command_cancel", (payload) => {
3220
- console.log("[TourClient] Received command_cancel:", payload);
3221
- if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3240
+ }
3241
+ });
3242
+ socket.on("tour:command", async (payload) => {
3243
+ const emitIfOpen = (ev, data) => {
3244
+ if (socketRef.current !== socket) return;
3245
+ if (!socket.connected) return;
3246
+ socket.emit(ev, data);
3247
+ };
3248
+ console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3249
+ runCleanup(pendingManualWaitCleanupRef.current);
3250
+ pendingManualWaitCleanupRef.current = null;
3251
+ if (voiceInputResolveRef.current) {
3252
+ const resolvePendingVoiceInput = voiceInputResolveRef.current;
3253
+ voiceInputResolveRef.current = null;
3254
+ resolvePendingVoiceInput("");
3255
+ }
3256
+ setPlaybackState("executing");
3257
+ commandInFlightRef.current = true;
3258
+ const commandBatchId = payload.commandBatchId ?? null;
3259
+ turnIdRef.current = payload.turnId ?? turnIdRef.current;
3260
+ const clearCommandBatchId = () => {
3261
+ if (activeCommandBatchIdRef.current === commandBatchId) {
3222
3262
  activeCommandBatchIdRef.current = null;
3223
- activeExecutionTokenRef.current++;
3224
- commandInFlightRef.current = false;
3225
- setPlaybackState("idle");
3226
- if (typeof window !== "undefined" && window.speechSynthesis) {
3227
- window.speechSynthesis.cancel();
3228
- }
3229
3263
  }
3230
- });
3231
- socket.on("tour:command", async (payload) => {
3232
- console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3233
- runCleanup(pendingManualWaitCleanupRef.current);
3234
- pendingManualWaitCleanupRef.current = null;
3235
- if (voiceInputResolveRef.current) {
3236
- const resolvePendingVoiceInput = voiceInputResolveRef.current;
3237
- voiceInputResolveRef.current = null;
3238
- resolvePendingVoiceInput("");
3239
- }
3240
- setPlaybackState("executing");
3241
- commandInFlightRef.current = true;
3242
- const commandBatchId = payload.commandBatchId ?? null;
3243
- turnIdRef.current = payload.turnId ?? turnIdRef.current;
3244
- const clearCommandBatchId = () => {
3245
- if (activeCommandBatchIdRef.current === commandBatchId) {
3246
- activeCommandBatchIdRef.current = null;
3264
+ };
3265
+ activeCommandBatchIdRef.current = commandBatchId;
3266
+ const executionToken = ++activeExecutionTokenRef.current;
3267
+ const activeTourId = tourRef.current?.id;
3268
+ const activePreviewRunId = previewRunIdRef.current;
3269
+ if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3270
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3271
+ stepOrder: payload.stepIndex,
3272
+ eventType: "command_batch_received",
3273
+ payload: {
3274
+ commandBatchId,
3275
+ commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3276
+ },
3277
+ currentStepOrder: payload.stepIndex
3278
+ });
3279
+ }
3280
+ if (typeof payload.stepIndex === "number") {
3281
+ const prevStep = stepIndexRef.current;
3282
+ stepIndexRef.current = payload.stepIndex;
3283
+ setCurrentStepIndex(payload.stepIndex);
3284
+ if (payload.stepIndex !== prevStep) {
3285
+ const tour = tourRef.current;
3286
+ const total = tour?.steps?.length ?? 0;
3287
+ if (tour && total > 0) {
3288
+ onStepChangeRef.current?.(payload.stepIndex, total, tour);
3247
3289
  }
3248
- };
3249
- activeCommandBatchIdRef.current = commandBatchId;
3250
- const executionToken = ++activeExecutionTokenRef.current;
3251
- const activeTourId = tourRef.current?.id;
3252
- const activePreviewRunId = previewRunIdRef.current;
3253
- if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3290
+ }
3291
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3254
3292
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3255
3293
  stepOrder: payload.stepIndex,
3256
- eventType: "command_batch_received",
3294
+ eventType: "step_started",
3257
3295
  payload: {
3258
- commandBatchId,
3259
- commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3296
+ previousStepOrder: prevStep,
3297
+ stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3260
3298
  },
3261
3299
  currentStepOrder: payload.stepIndex
3262
3300
  });
3263
3301
  }
3264
- if (typeof payload.stepIndex === "number") {
3265
- const prevStep = stepIndexRef.current;
3266
- stepIndexRef.current = payload.stepIndex;
3267
- setCurrentStepIndex(payload.stepIndex);
3268
- if (payload.stepIndex !== prevStep) {
3269
- const tour = tourRef.current;
3270
- const total = tour?.steps?.length ?? 0;
3271
- if (tour && total > 0) {
3272
- onStepChangeRef.current?.(payload.stepIndex, total, tour);
3273
- }
3274
- }
3275
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3276
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3277
- stepOrder: payload.stepIndex,
3278
- eventType: "step_started",
3279
- payload: {
3280
- previousStepOrder: prevStep,
3281
- stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3282
- },
3283
- currentStepOrder: payload.stepIndex
3284
- });
3285
- }
3286
- }
3287
- if (!payload.commands || !Array.isArray(payload.commands)) {
3288
- console.warn("[TourClient] Payload commands is not an array:", payload);
3289
- commandInFlightRef.current = false;
3290
- socket.emit("tour:action_result", {
3291
- success: false,
3292
- reason: "invalid_commands",
3293
- commandBatchId,
3294
- runId: runIdRef.current,
3295
- turnId: turnIdRef.current
3296
- });
3297
- clearCommandBatchId();
3298
- return;
3302
+ }
3303
+ if (!payload.commands || !Array.isArray(payload.commands)) {
3304
+ console.warn("[TourClient] Payload commands is not an array:", payload);
3305
+ commandInFlightRef.current = false;
3306
+ emitIfOpen("tour:action_result", {
3307
+ success: false,
3308
+ reason: "invalid_commands",
3309
+ commandBatchId,
3310
+ runId: runIdRef.current,
3311
+ turnId: turnIdRef.current
3312
+ });
3313
+ clearCommandBatchId();
3314
+ return;
3315
+ }
3316
+ let shouldWait = false;
3317
+ const results = [];
3318
+ let batchPreferredWaitTarget = null;
3319
+ const assertNotInterrupted = () => {
3320
+ if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3321
+ const error = new Error("interrupted");
3322
+ error.code = "INTERRUPTED";
3323
+ throw error;
3299
3324
  }
3300
- let shouldWait = false;
3301
- const results = [];
3302
- let batchPreferredWaitTarget = null;
3303
- const assertNotInterrupted = () => {
3304
- if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3305
- const error = new Error("interrupted");
3306
- error.code = "INTERRUPTED";
3307
- throw error;
3308
- }
3309
- };
3310
- const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3311
- const fallbackHints = fallbackStep?.element ?? null;
3312
- return retryLookup({
3313
- timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3314
- pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3315
- onRetry: () => {
3316
- assertNotInterrupted();
3317
- },
3318
- resolve: async () => {
3319
- let targetEl = null;
3320
- if (params.uid) {
3321
- const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3322
- targetEl = getElementByUid2(params.uid);
3323
- }
3324
- if (!targetEl) {
3325
- targetEl = resolveElementFromHints({
3326
- fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3327
- testId: params.testId ?? fallbackHints?.testId,
3328
- textContaining: params.textContaining ?? fallbackHints?.textContaining
3329
- }, fallbackStep, tagStore);
3330
- }
3331
- return targetEl;
3332
- }
3333
- });
3334
- };
3335
- const executeOne = async (action) => {
3336
- assertNotInterrupted();
3337
- console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3338
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3339
- const executeTimeline = async (params = {}) => {
3340
- const segments = Array.isArray(params.segments) ? params.segments : [];
3341
- for (let index = 0; index < segments.length; index += 1) {
3342
- assertNotInterrupted();
3343
- const segment = segments[index];
3344
- const segmentText = (segment?.text ?? "").trim();
3345
- const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3346
- const events = Array.isArray(segment?.events) ? segment.events : [];
3347
- if (segmentDelayMs > 0) {
3348
- await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3349
- }
3350
- if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3351
- const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3352
- if (!gateTarget) {
3353
- throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3354
- }
3355
- await waitForUserAction(gateTarget, segment.gate.event);
3356
- }
3357
- if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3358
- showCaption(segmentText);
3359
- }
3360
- const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3361
- const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3362
- prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3363
- onNearEnd: nextSegmentText ? () => {
3364
- void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3365
- } : void 0
3366
- }) : Promise.resolve();
3367
- const eventsPromise = (async () => {
3368
- for (const event of events) {
3369
- assertNotInterrupted();
3370
- const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3371
- if (delayMs > 0) {
3372
- await new Promise((resolve) => setTimeout(resolve, delayMs));
3373
- }
3374
- if (event?.action) {
3375
- await executeOne(event.action);
3376
- }
3377
- }
3378
- })();
3379
- await Promise.all([speechPromise, eventsPromise]);
3380
- }
3381
- if (params.removeHighlightAtEnd !== false) {
3382
- removeHighlight();
3383
- }
3384
- if (showCaptionsRef.current && reviewModeRef.current) {
3385
- removeCaption();
3386
- }
3387
- return !!params.waitForInput;
3388
- };
3389
- if (action.type === "speak") {
3390
- const text = action.params?.text ?? "";
3391
- if (!text.trim()) {
3392
- return { result: null };
3325
+ };
3326
+ const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3327
+ const fallbackHints = fallbackStep?.element ?? null;
3328
+ return retryLookup({
3329
+ timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3330
+ pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3331
+ onRetry: () => {
3332
+ assertNotInterrupted();
3333
+ },
3334
+ resolve: async () => {
3335
+ let targetEl = null;
3336
+ if (params.uid) {
3337
+ const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3338
+ targetEl = getElementByUid2(params.uid);
3393
3339
  }
3394
- if (showCaptionsRef.current && reviewModeRef.current) {
3395
- showCaption(text);
3340
+ if (!targetEl) {
3341
+ targetEl = resolveElementFromHints({
3342
+ fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3343
+ testId: params.testId ?? fallbackHints?.testId,
3344
+ textContaining: params.textContaining ?? fallbackHints?.textContaining
3345
+ }, fallbackStep, tagStore);
3396
3346
  }
3397
- const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3398
- // Schedule narration immediately so the client keeps its speculative,
3399
- // low-latency behavior, but defer batch completion until playback settles.
3400
- waitForCompletion: false,
3401
- interrupt: action.params?.interrupt
3402
- });
3403
- return { result: text, settlePromise };
3347
+ return targetEl;
3404
3348
  }
3405
- if (action.type === "play_timeline") {
3406
- const timelineShouldWait = await executeTimeline(action.params);
3407
- if (timelineShouldWait) shouldWait = true;
3408
- return { result: "timeline_executed" };
3409
- }
3410
- if (action.type === "highlight_element") {
3411
- const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3412
- if (resolvedEl) {
3413
- if (isEditableWaitTarget(resolvedEl)) {
3414
- batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3349
+ });
3350
+ };
3351
+ const executeOne = async (action) => {
3352
+ assertNotInterrupted();
3353
+ console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3354
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3355
+ const executeTimeline = async (params = {}) => {
3356
+ const segments = Array.isArray(params.segments) ? params.segments : [];
3357
+ for (let index = 0; index < segments.length; index += 1) {
3358
+ assertNotInterrupted();
3359
+ const segment = segments[index];
3360
+ const segmentText = (segment?.text ?? "").trim();
3361
+ const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3362
+ const events = Array.isArray(segment?.events) ? segment.events : [];
3363
+ if (segmentDelayMs > 0) {
3364
+ await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3365
+ }
3366
+ if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3367
+ const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3368
+ if (!gateTarget) {
3369
+ throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3415
3370
  }
3416
- showHighlight(resolvedEl, action.params?.label || action.label);
3417
- resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3418
- return { result: "highlighted" };
3371
+ await waitForUserAction(gateTarget, segment.gate.event);
3419
3372
  }
3420
- if (!action.params?.optional) {
3421
- throw new Error(
3422
- `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3423
- );
3373
+ if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3374
+ showCaption(segmentText);
3424
3375
  }
3425
- return { result: "highlight_optional_miss" };
3376
+ const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3377
+ const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3378
+ prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3379
+ onNearEnd: nextSegmentText ? () => {
3380
+ void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3381
+ } : void 0
3382
+ }) : Promise.resolve();
3383
+ const eventsPromise = (async () => {
3384
+ for (const event of events) {
3385
+ assertNotInterrupted();
3386
+ const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3387
+ if (delayMs > 0) {
3388
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
3389
+ }
3390
+ if (event?.action) {
3391
+ await executeOne(event.action);
3392
+ }
3393
+ }
3394
+ })();
3395
+ await Promise.all([speechPromise, eventsPromise]);
3426
3396
  }
3427
- if (action.type === "remove_highlight") {
3397
+ if (params.removeHighlightAtEnd !== false) {
3428
3398
  removeHighlight();
3429
- return { result: "highlight_removed" };
3430
3399
  }
3431
- if (action.type === "click_element") {
3432
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3433
- if (!targetEl) {
3434
- if (action.params?.optional) return { result: "click_optional_miss" };
3435
- throw new Error(
3436
- `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3437
- );
3438
- }
3439
- removeHighlight();
3440
- await performInteractiveClick(targetEl);
3441
- const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3442
- await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3443
- return { result: "clicked" };
3400
+ if (showCaptionsRef.current && reviewModeRef.current) {
3401
+ removeCaption();
3444
3402
  }
3445
- if (action.type === "fill_input") {
3446
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3447
- if (!targetEl) {
3448
- throw new Error(
3449
- `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3450
- );
3451
- }
3452
- const value = typeof action.params?.value === "string" ? action.params.value : "";
3453
- batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3454
- await performInteractiveFill(targetEl, value);
3455
- return { result: value };
3403
+ return !!params.waitForInput;
3404
+ };
3405
+ if (action.type === "speak") {
3406
+ const text = action.params?.text ?? "";
3407
+ if (!text.trim()) {
3408
+ return { result: null };
3456
3409
  }
3457
- if (action.type === "take_screenshot") {
3458
- const html2canvasModule = await import("html2canvas");
3459
- const html2canvas2 = html2canvasModule.default;
3460
- const canvas = await html2canvas2(document.body, {
3461
- useCORS: true,
3462
- allowTaint: true,
3463
- scale: Math.min(window.devicePixelRatio, 2),
3464
- width: window.innerWidth,
3465
- height: window.innerHeight,
3466
- x: window.scrollX,
3467
- y: window.scrollY,
3468
- logging: false
3469
- });
3470
- return { result: canvas.toDataURL("image/png") };
3410
+ if (showCaptionsRef.current && reviewModeRef.current) {
3411
+ showCaption(text);
3471
3412
  }
3472
- if (action.type === "navigate_to_url") {
3473
- const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3474
- if (!nextUrl) {
3475
- throw new Error("navigate_to_url missing url");
3413
+ const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3414
+ // Schedule narration immediately so the client keeps its speculative,
3415
+ // low-latency behavior, but defer batch completion until playback settles.
3416
+ waitForCompletion: false,
3417
+ interrupt: action.params?.interrupt
3418
+ });
3419
+ return { result: text, settlePromise };
3420
+ }
3421
+ if (action.type === "play_timeline") {
3422
+ const timelineShouldWait = await executeTimeline(action.params);
3423
+ if (timelineShouldWait) shouldWait = true;
3424
+ return { result: "timeline_executed" };
3425
+ }
3426
+ if (action.type === "highlight_element") {
3427
+ const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3428
+ if (resolvedEl) {
3429
+ if (isEditableWaitTarget(resolvedEl)) {
3430
+ batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3476
3431
  }
3477
- await navigateToTourUrl(nextUrl);
3478
- const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3479
- await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3480
- return { result: nextUrl };
3432
+ showHighlight(resolvedEl, action.params?.label || action.label);
3433
+ resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3434
+ return { result: "highlighted" };
3481
3435
  }
3482
- if (action.type === "execute_agent_action") {
3483
- const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3484
- if (!agentSocketId) {
3485
- throw new Error("No socketId available for execute_agent_action");
3486
- }
3487
- const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3488
- const res = await fetch(url, {
3489
- method: "POST",
3490
- headers: { "Content-Type": "application/json" },
3491
- body: JSON.stringify({
3492
- command: action.params?.command,
3493
- socketId: agentSocketId
3494
- })
3495
- });
3496
- if (!res.ok) {
3497
- throw new Error(`execute_agent_action failed: ${await res.text()}`);
3498
- }
3499
- if (action.params?.wait !== false) {
3500
- await res.json();
3501
- }
3502
- const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3503
- await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
3504
- await syncAOM();
3505
- return { result: action.params?.command ?? "executed" };
3436
+ if (!action.params?.optional) {
3437
+ throw new Error(
3438
+ `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3439
+ );
3506
3440
  }
3507
- if (action.type === "wait_for_user_action") {
3508
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3509
- if (!targetEl) {
3510
- throw new Error("wait_for_user_action target not found");
3511
- }
3512
- const eventName = action.params?.event ?? "click";
3513
- await waitForUserAction(targetEl, eventName);
3514
- return { result: `waited_for_${eventName}` };
3441
+ return { result: "highlight_optional_miss" };
3442
+ }
3443
+ if (action.type === "remove_highlight") {
3444
+ removeHighlight();
3445
+ return { result: "highlight_removed" };
3446
+ }
3447
+ if (action.type === "click_element") {
3448
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3449
+ if (!targetEl) {
3450
+ if (action.params?.optional) return { result: "click_optional_miss" };
3451
+ throw new Error(
3452
+ `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3453
+ );
3515
3454
  }
3516
- if (action.type === "wait_for_input") {
3517
- shouldWait = true;
3518
- return { result: "waiting_for_input" };
3455
+ removeHighlight();
3456
+ await performInteractiveClick(targetEl);
3457
+ const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3458
+ await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3459
+ return { result: "clicked" };
3460
+ }
3461
+ if (action.type === "fill_input") {
3462
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3463
+ if (!targetEl) {
3464
+ throw new Error(
3465
+ `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3466
+ );
3519
3467
  }
3520
- if (action.type === "end_tour") {
3521
- handleTourEnd();
3522
- return { result: "ended" };
3468
+ const value = typeof action.params?.value === "string" ? action.params.value : "";
3469
+ batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3470
+ await performInteractiveFill(targetEl, value);
3471
+ return { result: value };
3472
+ }
3473
+ if (action.type === "take_screenshot") {
3474
+ const html2canvasModule = await import("html2canvas");
3475
+ const html2canvas2 = html2canvasModule.default;
3476
+ const canvas = await html2canvas2(document.body, {
3477
+ useCORS: true,
3478
+ allowTaint: true,
3479
+ scale: Math.min(window.devicePixelRatio, 2),
3480
+ width: window.innerWidth,
3481
+ height: window.innerHeight,
3482
+ x: window.scrollX,
3483
+ y: window.scrollY,
3484
+ logging: false
3485
+ });
3486
+ return { result: canvas.toDataURL("image/png") };
3487
+ }
3488
+ if (action.type === "navigate_to_url") {
3489
+ const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3490
+ if (!nextUrl) {
3491
+ throw new Error("navigate_to_url missing url");
3523
3492
  }
3524
- console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3525
- return { result: "unknown_action_skipped" };
3526
- };
3527
- try {
3528
- const resultsBuffer = new Array(payload.commands.length);
3529
- const pendingUIActions = [];
3530
- for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3531
- const command = payload.commands[commandIndex];
3532
- assertNotInterrupted();
3533
- const isTerminal = isTerminalAction(command);
3534
- if (isTerminal) {
3535
- await Promise.all(pendingUIActions);
3536
- pendingUIActions.length = 0;
3537
- }
3538
- const executionPromise = (async () => {
3539
- const execution = await executeOne(command);
3540
- await execution.settlePromise;
3541
- resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3542
- })();
3543
- if (isTerminal) {
3544
- await executionPromise;
3545
- } else {
3546
- pendingUIActions.push(executionPromise);
3547
- }
3493
+ await navigateToTourUrl(nextUrl);
3494
+ const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3495
+ await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3496
+ return { result: nextUrl };
3497
+ }
3498
+ if (action.type === "execute_agent_action") {
3499
+ const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3500
+ if (!agentSocketId) {
3501
+ throw new Error("No socketId available for execute_agent_action");
3548
3502
  }
3549
- await Promise.all(pendingUIActions);
3550
- resultsBuffer.forEach((res) => {
3551
- if (res) results.push(res);
3503
+ const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3504
+ const res = await fetch(url, {
3505
+ method: "POST",
3506
+ headers: { "Content-Type": "application/json" },
3507
+ body: JSON.stringify({
3508
+ command: action.params?.command,
3509
+ socketId: agentSocketId
3510
+ })
3552
3511
  });
3512
+ if (!res.ok) {
3513
+ throw new Error(`execute_agent_action failed: ${await res.text()}`);
3514
+ }
3515
+ if (action.params?.wait !== false) {
3516
+ await res.json();
3517
+ }
3518
+ const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3519
+ await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
3553
3520
  await syncAOM();
3554
- } catch (err) {
3555
- commandInFlightRef.current = false;
3556
- const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3557
- if (interrupted) {
3558
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3559
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3560
- stepOrder: stepIndexRef.current,
3561
- eventType: "command_batch_interrupted",
3562
- payload: {
3563
- commandBatchId,
3564
- partialResults: results
3565
- },
3566
- currentStepOrder: stepIndexRef.current
3567
- });
3568
- }
3569
- socket.emit("tour:action_result", {
3570
- success: true,
3571
- interrupted: true,
3572
- results,
3573
- commandBatchId,
3574
- runId: runIdRef.current,
3575
- turnId: turnIdRef.current
3576
- });
3577
- clearCommandBatchId();
3578
- return;
3521
+ return { result: action.params?.command ?? "executed" };
3522
+ }
3523
+ if (action.type === "wait_for_user_action") {
3524
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3525
+ if (!targetEl) {
3526
+ throw new Error("wait_for_user_action target not found");
3527
+ }
3528
+ const eventName = action.params?.event ?? "click";
3529
+ await waitForUserAction(targetEl, eventName);
3530
+ return { result: `waited_for_${eventName}` };
3531
+ }
3532
+ if (action.type === "wait_for_input") {
3533
+ shouldWait = true;
3534
+ return { result: "waiting_for_input" };
3535
+ }
3536
+ if (action.type === "end_tour") {
3537
+ handleTourEnd();
3538
+ return { result: "ended" };
3539
+ }
3540
+ console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3541
+ return { result: "unknown_action_skipped" };
3542
+ };
3543
+ try {
3544
+ const resultsBuffer = new Array(payload.commands.length);
3545
+ const pendingUIActions = [];
3546
+ for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3547
+ const command = payload.commands[commandIndex];
3548
+ assertNotInterrupted();
3549
+ const isTerminal = isTerminalAction(command);
3550
+ if (isTerminal) {
3551
+ await Promise.all(pendingUIActions);
3552
+ pendingUIActions.length = 0;
3579
3553
  }
3580
- console.error("[TourClient] Command batch execution failed:", err);
3554
+ const executionPromise = (async () => {
3555
+ const execution = await executeOne(command);
3556
+ await execution.settlePromise;
3557
+ resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3558
+ })();
3559
+ if (isTerminal) {
3560
+ await executionPromise;
3561
+ } else {
3562
+ pendingUIActions.push(executionPromise);
3563
+ }
3564
+ }
3565
+ await Promise.all(pendingUIActions);
3566
+ resultsBuffer.forEach((res) => {
3567
+ if (res) results.push(res);
3568
+ });
3569
+ await syncAOM();
3570
+ } catch (err) {
3571
+ commandInFlightRef.current = false;
3572
+ const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3573
+ if (interrupted) {
3581
3574
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3582
3575
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3583
3576
  stepOrder: stepIndexRef.current,
3584
- eventType: "command_batch_failed",
3577
+ eventType: "command_batch_interrupted",
3585
3578
  payload: {
3586
3579
  commandBatchId,
3587
- error: String(err),
3588
3580
  partialResults: results
3589
3581
  },
3590
- status: "active",
3591
3582
  currentStepOrder: stepIndexRef.current
3592
3583
  });
3593
3584
  }
3594
- socket.emit("tour:action_result", {
3595
- success: false,
3596
- reason: "execution_error",
3597
- error: String(err),
3585
+ emitIfOpen("tour:action_result", {
3586
+ success: true,
3587
+ interrupted: true,
3598
3588
  results,
3599
3589
  commandBatchId,
3600
3590
  runId: runIdRef.current,
@@ -3603,108 +3593,82 @@ function useTourPlayback({
3603
3593
  clearCommandBatchId();
3604
3594
  return;
3605
3595
  }
3606
- commandInFlightRef.current = false;
3607
- if (shouldWait && !skipRequestedRef.current) {
3608
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3609
- const waitCondition = currentStep?.onboarding?.waitCondition;
3610
- const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3611
- const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3612
- const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3613
- let manualWaitPromise = null;
3614
- let manualWaitKind = null;
3615
- const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3616
- const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3617
- runCleanup(pendingManualWaitCleanupRef.current);
3618
- pendingManualWaitCleanupRef.current = null;
3619
- if (waitTargetHints) {
3620
- let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3621
- if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3622
- manualWaitTarget = preferredWaitTarget;
3623
- console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3624
- }
3625
- if (manualWaitTarget) {
3626
- const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3627
- manualWaitPromise = manualWait.promise;
3628
- manualWaitKind = manualWait.kind;
3629
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3630
- }
3596
+ console.error("[TourClient] Command batch execution failed:", err);
3597
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3598
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3599
+ stepOrder: stepIndexRef.current,
3600
+ eventType: "command_batch_failed",
3601
+ payload: {
3602
+ commandBatchId,
3603
+ error: String(err),
3604
+ partialResults: results
3605
+ },
3606
+ status: "active",
3607
+ currentStepOrder: stepIndexRef.current
3608
+ });
3609
+ }
3610
+ emitIfOpen("tour:action_result", {
3611
+ success: false,
3612
+ reason: "execution_error",
3613
+ error: String(err),
3614
+ results,
3615
+ commandBatchId,
3616
+ runId: runIdRef.current,
3617
+ turnId: turnIdRef.current
3618
+ });
3619
+ clearCommandBatchId();
3620
+ return;
3621
+ }
3622
+ commandInFlightRef.current = false;
3623
+ if (shouldWait && !skipRequestedRef.current) {
3624
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3625
+ const waitCondition = currentStep?.onboarding?.waitCondition;
3626
+ const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3627
+ const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3628
+ const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3629
+ let manualWaitPromise = null;
3630
+ let manualWaitKind = null;
3631
+ const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3632
+ const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3633
+ runCleanup(pendingManualWaitCleanupRef.current);
3634
+ pendingManualWaitCleanupRef.current = null;
3635
+ if (waitTargetHints) {
3636
+ let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3637
+ if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3638
+ manualWaitTarget = preferredWaitTarget;
3639
+ console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3631
3640
  }
3632
- if (!manualWaitPromise && preferredWaitTarget) {
3633
- const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3641
+ if (manualWaitTarget) {
3642
+ const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3634
3643
  manualWaitPromise = manualWait.promise;
3635
3644
  manualWaitKind = manualWait.kind;
3636
3645
  pendingManualWaitCleanupRef.current = manualWait.cleanup;
3637
- console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3638
3646
  }
3639
- if (!manualWaitPromise && inputLikeWait) {
3640
- const firstInput = document.querySelector(
3641
- 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3642
- );
3643
- if (firstInput) {
3644
- const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3645
- manualWaitPromise = manualWait.promise;
3646
- manualWaitKind = manualWait.kind;
3647
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3648
- console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3649
- }
3650
- }
3651
- setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3652
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3653
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3654
- stepOrder: stepIndexRef.current,
3655
- eventType: "waiting_for_input",
3656
- payload: {
3657
- commandBatchId,
3658
- results
3659
- },
3660
- currentStepOrder: stepIndexRef.current
3661
- });
3647
+ }
3648
+ if (!manualWaitPromise && preferredWaitTarget) {
3649
+ const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3650
+ manualWaitPromise = manualWait.promise;
3651
+ manualWaitKind = manualWait.kind;
3652
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3653
+ console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3654
+ }
3655
+ if (!manualWaitPromise && inputLikeWait) {
3656
+ const firstInput = document.querySelector(
3657
+ 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3658
+ );
3659
+ if (firstInput) {
3660
+ const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3661
+ manualWaitPromise = manualWait.promise;
3662
+ manualWaitKind = manualWait.kind;
3663
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3664
+ console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3662
3665
  }
3663
- socket.emit("tour:action_result", {
3664
- success: true,
3665
- waitingForInput: true,
3666
- results,
3667
- commandBatchId,
3668
- runId: runIdRef.current,
3669
- turnId: turnIdRef.current
3670
- });
3671
- clearCommandBatchId();
3672
- const voiceOrTextWaitPromise = new Promise((resolve) => {
3673
- if (pendingInputBufRef.current) {
3674
- const flushed = pendingInputBufRef.current;
3675
- pendingInputBufRef.current = null;
3676
- resolve(flushed);
3677
- return;
3678
- }
3679
- voiceInputResolveRef.current = (text) => {
3680
- voiceInputResolveRef.current = null;
3681
- resolve(text);
3682
- };
3683
- });
3684
- Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3685
- runCleanup(pendingManualWaitCleanupRef.current);
3686
- pendingManualWaitCleanupRef.current = null;
3687
- voiceInputResolveRef.current = null;
3688
- setPlaybackState("executing");
3689
- const transcript = userText.trim();
3690
- if (!transcript) {
3691
- return;
3692
- }
3693
- const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3694
- await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
3695
- await syncAOM();
3696
- socket.emit("tour:user_input", {
3697
- transcript,
3698
- runId: runIdRef.current,
3699
- turnId: turnIdRef.current
3700
- });
3701
- });
3702
- return;
3703
3666
  }
3667
+ setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3704
3668
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3705
3669
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3706
3670
  stepOrder: stepIndexRef.current,
3707
- eventType: "command_batch_completed",
3671
+ eventType: "waiting_for_input",
3708
3672
  payload: {
3709
3673
  commandBatchId,
3710
3674
  results
@@ -3712,101 +3676,169 @@ function useTourPlayback({
3712
3676
  currentStepOrder: stepIndexRef.current
3713
3677
  });
3714
3678
  }
3715
- socket.emit("tour:action_result", {
3679
+ emitIfOpen("tour:action_result", {
3716
3680
  success: true,
3681
+ waitingForInput: true,
3717
3682
  results,
3718
3683
  commandBatchId,
3719
3684
  runId: runIdRef.current,
3720
3685
  turnId: turnIdRef.current
3721
3686
  });
3722
3687
  clearCommandBatchId();
3723
- });
3724
- socket.on("tour:start", async (tourData) => {
3725
- if (isActiveRef.current) return;
3726
- runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3727
- const tour = tourData.tourContext ?? tourRef.current;
3728
- if (tour?.type && tour.type !== experienceType) {
3729
- console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${experienceType})`);
3730
- return;
3731
- }
3732
- skipRequestedRef.current = false;
3733
- const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3734
- isActiveRef.current = true;
3735
- setIsActive(true);
3736
- setActiveTour(tour ?? null);
3737
- tourRef.current = tour ?? null;
3738
- setTotalSteps(total);
3739
- stepIndexRef.current = 0;
3740
- setCurrentStepIndex(0);
3741
- setPlaybackState("intro");
3742
- if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3743
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3744
- stepOrder: 0,
3745
- eventType: "tour_started",
3746
- payload: {
3747
- totalSteps: total,
3748
- source: "sdk_test_preview"
3749
- },
3750
- currentStepOrder: 0
3688
+ const voiceOrTextWaitPromise = new Promise((resolve) => {
3689
+ if (pendingInputBufRef.current) {
3690
+ const flushed = pendingInputBufRef.current;
3691
+ pendingInputBufRef.current = null;
3692
+ resolve(flushed);
3693
+ return;
3694
+ }
3695
+ voiceInputResolveRef.current = (text) => {
3696
+ voiceInputResolveRef.current = null;
3697
+ resolve(text);
3698
+ };
3699
+ });
3700
+ Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3701
+ runCleanup(pendingManualWaitCleanupRef.current);
3702
+ pendingManualWaitCleanupRef.current = null;
3703
+ voiceInputResolveRef.current = null;
3704
+ setPlaybackState("executing");
3705
+ const transcript = userText.trim();
3706
+ if (!transcript) {
3707
+ return;
3708
+ }
3709
+ const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3710
+ await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
3711
+ await syncAOM();
3712
+ emitIfOpen("tour:user_input", {
3713
+ transcript,
3714
+ runId: runIdRef.current,
3715
+ turnId: turnIdRef.current
3751
3716
  });
3752
- }
3753
- try {
3754
- const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3755
- const aom = generateMinifiedAOM2();
3717
+ });
3718
+ return;
3719
+ }
3720
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3721
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3722
+ stepOrder: stepIndexRef.current,
3723
+ eventType: "command_batch_completed",
3724
+ payload: {
3725
+ commandBatchId,
3726
+ results
3727
+ },
3728
+ currentStepOrder: stepIndexRef.current
3729
+ });
3730
+ }
3731
+ emitIfOpen("tour:action_result", {
3732
+ success: true,
3733
+ results,
3734
+ commandBatchId,
3735
+ runId: runIdRef.current,
3736
+ turnId: turnIdRef.current
3737
+ });
3738
+ clearCommandBatchId();
3739
+ });
3740
+ socket.on("tour:start", async (tourData) => {
3741
+ if (isActiveRef.current) return;
3742
+ runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3743
+ const tour = tourData.tourContext ?? tourRef.current;
3744
+ const expType = experienceTypeRef.current;
3745
+ if (tour?.type && tour.type !== expType) {
3746
+ console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3747
+ return;
3748
+ }
3749
+ skipRequestedRef.current = false;
3750
+ const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3751
+ isActiveRef.current = true;
3752
+ setIsActive(true);
3753
+ setActiveTour(tour ?? null);
3754
+ tourRef.current = tour ?? null;
3755
+ setTotalSteps(total);
3756
+ stepIndexRef.current = 0;
3757
+ setCurrentStepIndex(0);
3758
+ setPlaybackState("intro");
3759
+ if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3760
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3761
+ stepOrder: 0,
3762
+ eventType: "tour_started",
3763
+ payload: {
3764
+ totalSteps: total,
3765
+ source: "sdk_test_preview"
3766
+ },
3767
+ currentStepOrder: 0
3768
+ });
3769
+ }
3770
+ try {
3771
+ const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3772
+ const aom = generateMinifiedAOM2();
3773
+ if (socketRef.current === socket && socket.connected) {
3756
3774
  socket.emit("tour:sync_dom", {
3757
3775
  url: window.location.pathname + window.location.search + window.location.hash,
3758
3776
  aom: aom.nodes,
3759
3777
  domSummary: captureDomSummary()
3760
3778
  });
3761
- } catch (e) {
3762
- console.warn("[TourClient] Initial DOM sync failed:", e);
3763
3779
  }
3764
- });
3765
- socket.on("tour:update", (payload) => {
3766
- const updatedTour = payload?.tourContext;
3767
- if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3768
- return;
3769
- }
3770
- tourRef.current = updatedTour;
3771
- setActiveTour(updatedTour);
3772
- const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3773
- setTotalSteps(nextTotal);
3774
- if (typeof payload.currentStepIndex === "number") {
3775
- const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3776
- stepIndexRef.current = clampedStepIndex;
3777
- setCurrentStepIndex(clampedStepIndex);
3778
- if (nextTotal > 0) {
3779
- onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3780
- }
3780
+ } catch (e) {
3781
+ console.warn("[TourClient] Initial DOM sync failed:", e);
3782
+ }
3783
+ });
3784
+ socket.on("tour:update", (payload) => {
3785
+ const updatedTour = payload?.tourContext;
3786
+ if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3787
+ return;
3788
+ }
3789
+ tourRef.current = updatedTour;
3790
+ setActiveTour(updatedTour);
3791
+ const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3792
+ setTotalSteps(nextTotal);
3793
+ if (typeof payload.currentStepIndex === "number") {
3794
+ const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3795
+ stepIndexRef.current = clampedStepIndex;
3796
+ setCurrentStepIndex(clampedStepIndex);
3797
+ if (nextTotal > 0) {
3798
+ onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3781
3799
  }
3782
- });
3783
- socket.on("tour:end", () => {
3784
- setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3785
- handleTourEnd();
3786
- });
3787
- socket.on("tour:debug_log", (entry) => {
3788
- const isDev = devMode || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3789
- if (isDev) {
3790
- console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3791
- if (typeof window !== "undefined") {
3792
- window.dispatchEvent(new CustomEvent("modelnex-debug", {
3793
- detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3794
- }));
3795
- }
3800
+ }
3801
+ });
3802
+ socket.on("tour:end", () => {
3803
+ setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3804
+ handleTourEnd();
3805
+ });
3806
+ socket.on("tour:debug_log", (entry) => {
3807
+ const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3808
+ if (isDev) {
3809
+ console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3810
+ if (typeof window !== "undefined") {
3811
+ window.dispatchEvent(new CustomEvent("modelnex-debug", {
3812
+ detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3813
+ }));
3796
3814
  }
3797
- });
3798
- console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devMode || process.env.NODE_ENV === "development");
3815
+ }
3799
3816
  });
3817
+ console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3800
3818
  return () => {
3801
- if (socketRef.current) {
3802
- socketRef.current.disconnect();
3803
- socketRef.current = null;
3819
+ const toClose = createdSocket ?? socketRef.current;
3820
+ if (toClose) {
3821
+ toClose.disconnect();
3804
3822
  }
3823
+ createdSocket = null;
3824
+ socketRef.current = null;
3805
3825
  setServerState(null);
3806
3826
  runIdRef.current = null;
3807
3827
  turnIdRef.current = null;
3808
3828
  };
3809
- }, [serverUrl, websiteId, userProfile, disabled, devMode, experienceType]);
3829
+ }, [serverUrl, websiteId, disabled]);
3830
+ (0, import_react12.useEffect)(() => {
3831
+ if (disabled) return;
3832
+ const s = socketRef.current;
3833
+ const profile = userProfile;
3834
+ if (!s?.connected || !websiteId || !profile?.userId) return;
3835
+ const timer = setTimeout(() => {
3836
+ if (s.connected) {
3837
+ s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3838
+ }
3839
+ }, 150);
3840
+ return () => clearTimeout(timer);
3841
+ }, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
3810
3842
  (0, import_react12.useEffect)(() => {
3811
3843
  if (!showCaptions || !isReviewMode) {
3812
3844
  removeCaption();
@@ -3830,11 +3862,13 @@ function useTourPlayback({
3830
3862
  if (!socketRef.current?.connected || !isActiveRef.current) return;
3831
3863
  const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3832
3864
  const aom = generateMinifiedAOM2();
3833
- socketRef.current.emit("tour:sync_dom", {
3834
- url: window.location.pathname + window.location.search + window.location.hash,
3835
- aom: aom.nodes,
3836
- domSummary: captureDomSummary()
3837
- });
3865
+ if (socketRef.current?.connected) {
3866
+ socketRef.current.emit("tour:sync_dom", {
3867
+ url: window.location.pathname + window.location.search + window.location.hash,
3868
+ aom: aom.nodes,
3869
+ domSummary: captureDomSummary()
3870
+ });
3871
+ }
3838
3872
  }, []);
3839
3873
  const interruptExecution = (0, import_react12.useCallback)((transcript) => {
3840
3874
  if (!socketRef.current?.connected || !isActiveRef.current) return false;
@@ -3845,21 +3879,23 @@ function useTourPlayback({
3845
3879
  removeHighlight();
3846
3880
  removeCaption();
3847
3881
  voice.stopSpeaking();
3848
- socketRef.current.emit("tour:action_result", {
3849
- success: true,
3850
- interrupted: true,
3851
- transcript,
3852
- commandBatchId: activeCommandBatchIdRef.current,
3853
- runId: runIdRef.current,
3854
- turnId: turnIdRef.current
3855
- });
3856
- activeCommandBatchIdRef.current = null;
3857
- socketRef.current.emit("tour:user_input", {
3858
- transcript,
3859
- interrupted: true,
3860
- runId: runIdRef.current,
3861
- turnId: turnIdRef.current
3862
- });
3882
+ if (socketRef.current?.connected) {
3883
+ socketRef.current.emit("tour:action_result", {
3884
+ success: true,
3885
+ interrupted: true,
3886
+ transcript,
3887
+ commandBatchId: activeCommandBatchIdRef.current,
3888
+ runId: runIdRef.current,
3889
+ turnId: turnIdRef.current
3890
+ });
3891
+ activeCommandBatchIdRef.current = null;
3892
+ socketRef.current.emit("tour:user_input", {
3893
+ transcript,
3894
+ interrupted: true,
3895
+ runId: runIdRef.current,
3896
+ turnId: turnIdRef.current
3897
+ });
3898
+ }
3863
3899
  setPlaybackState("thinking");
3864
3900
  return true;
3865
3901
  }, [voice]);
@@ -3977,11 +4013,13 @@ function useTourPlayback({
3977
4013
  previewRunIdRef.current = null;
3978
4014
  }
3979
4015
  tourRef.current = tour;
3980
- socketRef.current.emit("tour:request_start", {
3981
- tourId: tour.id,
3982
- previewRunId: previewRunIdRef.current,
3983
- tourContext: tour
3984
- });
4016
+ if (socketRef.current?.connected) {
4017
+ socketRef.current.emit("tour:request_start", {
4018
+ tourId: tour.id,
4019
+ previewRunId: previewRunIdRef.current,
4020
+ tourContext: tour
4021
+ });
4022
+ }
3985
4023
  }, [serverUrl, websiteId]);
3986
4024
  (0, import_react12.useEffect)(() => {
3987
4025
  if (!enableAutoDiscovery) return;
@@ -5429,7 +5467,7 @@ function useVoice(serverUrl) {
5429
5467
  (async () => {
5430
5468
  try {
5431
5469
  const ioModule = await import("socket.io-client");
5432
- const io2 = ioModule.default || ioModule;
5470
+ const io3 = ioModule.default || ioModule;
5433
5471
  const stream = await navigator.mediaDevices.getUserMedia({
5434
5472
  audio: {
5435
5473
  echoCancellation: true,
@@ -5451,7 +5489,7 @@ function useVoice(serverUrl) {
5451
5489
  audioBitsPerSecond: 128e3
5452
5490
  });
5453
5491
  mediaRecorderRef.current = recorder;
5454
- const socket = io2(serverUrl, {
5492
+ const socket = io3(serverUrl, {
5455
5493
  path: "/socket.io",
5456
5494
  transports: resolveSocketIoTransports(serverUrl, "websocket-first")
5457
5495
  });
@@ -10972,7 +11010,7 @@ var ModelNexProvider = ({
10972
11010
  socketId,
10973
11011
  devMode
10974
11012
  }),
10975
- [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
11013
+ [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
10976
11014
  );
10977
11015
  return import_react21.default.createElement(
10978
11016
  ModelNexContext.Provider,