@modelnex/sdk 0.5.15 → 0.5.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -2626,6 +2626,7 @@ function isTourEligible(tour, userProfile) {
2626
2626
 
2627
2627
  // src/hooks/useTourPlayback.ts
2628
2628
  var import_react12 = require("react");
2629
+ var import_socket2 = require("socket.io-client");
2629
2630
 
2630
2631
  // src/utils/retryLookup.ts
2631
2632
  var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
@@ -3199,629 +3200,622 @@ function useTourPlayback({
3199
3200
  (0, import_react12.useEffect)(() => {
3200
3201
  if (disabled) return;
3201
3202
  if (typeof window === "undefined") return;
3202
- let cancelled = false;
3203
3203
  let createdSocket = null;
3204
- import("socket.io-client").then((ioModule) => {
3205
- if (cancelled) return;
3206
- const io2 = ioModule.default || ioModule;
3207
- const socket = io2(serverUrl, {
3208
- path: "/socket.io",
3209
- // standard
3210
- transports: resolveSocketIoTransports(serverUrl, "polling-first")
3211
- });
3212
- if (cancelled) {
3213
- socket.disconnect();
3214
- return;
3204
+ const socket = (0, import_socket2.io)(serverUrl, {
3205
+ path: "/socket.io",
3206
+ transports: resolveSocketIoTransports(serverUrl, "polling-first"),
3207
+ autoConnect: true,
3208
+ reconnection: true,
3209
+ reconnectionAttempts: 10,
3210
+ reconnectionDelay: 1e3
3211
+ });
3212
+ createdSocket = socket;
3213
+ socketRef.current = socket;
3214
+ socket.on("connect", () => {
3215
+ console.log("[TourClient] Connected to tour agent server:", socket.id);
3216
+ const profile = userProfileRef.current;
3217
+ if (websiteId && profile?.userId && socket.connected) {
3218
+ socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3215
3219
  }
3216
- createdSocket = socket;
3217
- socketRef.current = socket;
3218
- socket.on("connect", () => {
3219
- console.log("[TourClient] Connected to tour agent server:", socket.id);
3220
- const profile = userProfileRef.current;
3221
- if (websiteId && profile?.userId) {
3222
- socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3223
- }
3224
- });
3225
- socket.on("tour:server_state", (payload) => {
3226
- if (typeof payload?.runId === "number") {
3227
- runIdRef.current = payload.runId;
3228
- }
3229
- if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3230
- turnIdRef.current = payload.turnId ?? null;
3220
+ });
3221
+ socket.on("tour:server_state", (payload) => {
3222
+ if (typeof payload?.runId === "number") {
3223
+ runIdRef.current = payload.runId;
3224
+ }
3225
+ if (typeof payload?.turnId === "string" || payload?.turnId === null) {
3226
+ turnIdRef.current = payload.turnId ?? null;
3227
+ }
3228
+ setServerState(payload);
3229
+ });
3230
+ socket.on("tour:command_cancel", (payload) => {
3231
+ console.log("[TourClient] Received command_cancel:", payload);
3232
+ if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3233
+ activeCommandBatchIdRef.current = null;
3234
+ activeExecutionTokenRef.current++;
3235
+ commandInFlightRef.current = false;
3236
+ setPlaybackState("idle");
3237
+ if (typeof window !== "undefined" && window.speechSynthesis) {
3238
+ window.speechSynthesis.cancel();
3231
3239
  }
3232
- setServerState(payload);
3233
- });
3234
- socket.on("tour:command_cancel", (payload) => {
3235
- console.log("[TourClient] Received command_cancel:", payload);
3236
- if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
3240
+ }
3241
+ });
3242
+ socket.on("tour:command", async (payload) => {
3243
+ const emitIfOpen = (ev, data) => {
3244
+ if (socketRef.current !== socket) return;
3245
+ if (!socket.connected) return;
3246
+ socket.emit(ev, data);
3247
+ };
3248
+ console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3249
+ runCleanup(pendingManualWaitCleanupRef.current);
3250
+ pendingManualWaitCleanupRef.current = null;
3251
+ if (voiceInputResolveRef.current) {
3252
+ const resolvePendingVoiceInput = voiceInputResolveRef.current;
3253
+ voiceInputResolveRef.current = null;
3254
+ resolvePendingVoiceInput("");
3255
+ }
3256
+ setPlaybackState("executing");
3257
+ commandInFlightRef.current = true;
3258
+ const commandBatchId = payload.commandBatchId ?? null;
3259
+ turnIdRef.current = payload.turnId ?? turnIdRef.current;
3260
+ const clearCommandBatchId = () => {
3261
+ if (activeCommandBatchIdRef.current === commandBatchId) {
3237
3262
  activeCommandBatchIdRef.current = null;
3238
- activeExecutionTokenRef.current++;
3239
- commandInFlightRef.current = false;
3240
- setPlaybackState("idle");
3241
- if (typeof window !== "undefined" && window.speechSynthesis) {
3242
- window.speechSynthesis.cancel();
3243
- }
3244
3263
  }
3245
- });
3246
- socket.on("tour:command", async (payload) => {
3247
- const emitIfOpen = (ev, data) => {
3248
- if (socketRef.current !== socket) return;
3249
- if (!socket.connected) return;
3250
- socket.emit(ev, data);
3251
- };
3252
- console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
3253
- runCleanup(pendingManualWaitCleanupRef.current);
3254
- pendingManualWaitCleanupRef.current = null;
3255
- if (voiceInputResolveRef.current) {
3256
- const resolvePendingVoiceInput = voiceInputResolveRef.current;
3257
- voiceInputResolveRef.current = null;
3258
- resolvePendingVoiceInput("");
3259
- }
3260
- setPlaybackState("executing");
3261
- commandInFlightRef.current = true;
3262
- const commandBatchId = payload.commandBatchId ?? null;
3263
- turnIdRef.current = payload.turnId ?? turnIdRef.current;
3264
- const clearCommandBatchId = () => {
3265
- if (activeCommandBatchIdRef.current === commandBatchId) {
3266
- activeCommandBatchIdRef.current = null;
3264
+ };
3265
+ activeCommandBatchIdRef.current = commandBatchId;
3266
+ const executionToken = ++activeExecutionTokenRef.current;
3267
+ const activeTourId = tourRef.current?.id;
3268
+ const activePreviewRunId = previewRunIdRef.current;
3269
+ if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3270
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3271
+ stepOrder: payload.stepIndex,
3272
+ eventType: "command_batch_received",
3273
+ payload: {
3274
+ commandBatchId,
3275
+ commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3276
+ },
3277
+ currentStepOrder: payload.stepIndex
3278
+ });
3279
+ }
3280
+ if (typeof payload.stepIndex === "number") {
3281
+ const prevStep = stepIndexRef.current;
3282
+ stepIndexRef.current = payload.stepIndex;
3283
+ setCurrentStepIndex(payload.stepIndex);
3284
+ if (payload.stepIndex !== prevStep) {
3285
+ const tour = tourRef.current;
3286
+ const total = tour?.steps?.length ?? 0;
3287
+ if (tour && total > 0) {
3288
+ onStepChangeRef.current?.(payload.stepIndex, total, tour);
3267
3289
  }
3268
- };
3269
- activeCommandBatchIdRef.current = commandBatchId;
3270
- const executionToken = ++activeExecutionTokenRef.current;
3271
- const activeTourId = tourRef.current?.id;
3272
- const activePreviewRunId = previewRunIdRef.current;
3273
- if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
3290
+ }
3291
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3274
3292
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3275
3293
  stepOrder: payload.stepIndex,
3276
- eventType: "command_batch_received",
3294
+ eventType: "step_started",
3277
3295
  payload: {
3278
- commandBatchId,
3279
- commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
3296
+ previousStepOrder: prevStep,
3297
+ stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3280
3298
  },
3281
3299
  currentStepOrder: payload.stepIndex
3282
3300
  });
3283
3301
  }
3284
- if (typeof payload.stepIndex === "number") {
3285
- const prevStep = stepIndexRef.current;
3286
- stepIndexRef.current = payload.stepIndex;
3287
- setCurrentStepIndex(payload.stepIndex);
3288
- if (payload.stepIndex !== prevStep) {
3289
- const tour = tourRef.current;
3290
- const total = tour?.steps?.length ?? 0;
3291
- if (tour && total > 0) {
3292
- onStepChangeRef.current?.(payload.stepIndex, total, tour);
3293
- }
3294
- }
3295
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3296
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3297
- stepOrder: payload.stepIndex,
3298
- eventType: "step_started",
3299
- payload: {
3300
- previousStepOrder: prevStep,
3301
- stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
3302
- },
3303
- currentStepOrder: payload.stepIndex
3304
- });
3305
- }
3306
- }
3307
- if (!payload.commands || !Array.isArray(payload.commands)) {
3308
- console.warn("[TourClient] Payload commands is not an array:", payload);
3309
- commandInFlightRef.current = false;
3310
- emitIfOpen("tour:action_result", {
3311
- success: false,
3312
- reason: "invalid_commands",
3313
- commandBatchId,
3314
- runId: runIdRef.current,
3315
- turnId: turnIdRef.current
3316
- });
3317
- clearCommandBatchId();
3318
- return;
3302
+ }
3303
+ if (!payload.commands || !Array.isArray(payload.commands)) {
3304
+ console.warn("[TourClient] Payload commands is not an array:", payload);
3305
+ commandInFlightRef.current = false;
3306
+ emitIfOpen("tour:action_result", {
3307
+ success: false,
3308
+ reason: "invalid_commands",
3309
+ commandBatchId,
3310
+ runId: runIdRef.current,
3311
+ turnId: turnIdRef.current
3312
+ });
3313
+ clearCommandBatchId();
3314
+ return;
3315
+ }
3316
+ let shouldWait = false;
3317
+ const results = [];
3318
+ let batchPreferredWaitTarget = null;
3319
+ const assertNotInterrupted = () => {
3320
+ if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3321
+ const error = new Error("interrupted");
3322
+ error.code = "INTERRUPTED";
3323
+ throw error;
3319
3324
  }
3320
- let shouldWait = false;
3321
- const results = [];
3322
- let batchPreferredWaitTarget = null;
3323
- const assertNotInterrupted = () => {
3324
- if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
3325
- const error = new Error("interrupted");
3326
- error.code = "INTERRUPTED";
3327
- throw error;
3328
- }
3329
- };
3330
- const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3331
- const fallbackHints = fallbackStep?.element ?? null;
3332
- return retryLookup({
3333
- timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3334
- pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3335
- onRetry: () => {
3336
- assertNotInterrupted();
3337
- },
3338
- resolve: async () => {
3339
- let targetEl = null;
3340
- if (params.uid) {
3341
- const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3342
- targetEl = getElementByUid2(params.uid);
3343
- }
3344
- if (!targetEl) {
3345
- targetEl = resolveElementFromHints({
3346
- fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3347
- testId: params.testId ?? fallbackHints?.testId,
3348
- textContaining: params.textContaining ?? fallbackHints?.textContaining
3349
- }, fallbackStep, tagStore);
3350
- }
3351
- return targetEl;
3352
- }
3353
- });
3354
- };
3355
- const executeOne = async (action) => {
3356
- assertNotInterrupted();
3357
- console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3358
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3359
- const executeTimeline = async (params = {}) => {
3360
- const segments = Array.isArray(params.segments) ? params.segments : [];
3361
- for (let index = 0; index < segments.length; index += 1) {
3362
- assertNotInterrupted();
3363
- const segment = segments[index];
3364
- const segmentText = (segment?.text ?? "").trim();
3365
- const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3366
- const events = Array.isArray(segment?.events) ? segment.events : [];
3367
- if (segmentDelayMs > 0) {
3368
- await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3369
- }
3370
- if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3371
- const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3372
- if (!gateTarget) {
3373
- throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3374
- }
3375
- await waitForUserAction(gateTarget, segment.gate.event);
3376
- }
3377
- if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3378
- showCaption(segmentText);
3379
- }
3380
- const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3381
- const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3382
- prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3383
- onNearEnd: nextSegmentText ? () => {
3384
- void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3385
- } : void 0
3386
- }) : Promise.resolve();
3387
- const eventsPromise = (async () => {
3388
- for (const event of events) {
3389
- assertNotInterrupted();
3390
- const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3391
- if (delayMs > 0) {
3392
- await new Promise((resolve) => setTimeout(resolve, delayMs));
3393
- }
3394
- if (event?.action) {
3395
- await executeOne(event.action);
3396
- }
3397
- }
3398
- })();
3399
- await Promise.all([speechPromise, eventsPromise]);
3400
- }
3401
- if (params.removeHighlightAtEnd !== false) {
3402
- removeHighlight();
3403
- }
3404
- if (showCaptionsRef.current && reviewModeRef.current) {
3405
- removeCaption();
3406
- }
3407
- return !!params.waitForInput;
3408
- };
3409
- if (action.type === "speak") {
3410
- const text = action.params?.text ?? "";
3411
- if (!text.trim()) {
3412
- return { result: null };
3325
+ };
3326
+ const resolveTargetElement2 = async (params = {}, fallbackStep) => {
3327
+ const fallbackHints = fallbackStep?.element ?? null;
3328
+ return retryLookup({
3329
+ timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
3330
+ pollMs: Math.max(50, Number(params.pollMs ?? 120)),
3331
+ onRetry: () => {
3332
+ assertNotInterrupted();
3333
+ },
3334
+ resolve: async () => {
3335
+ let targetEl = null;
3336
+ if (params.uid) {
3337
+ const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3338
+ targetEl = getElementByUid2(params.uid);
3413
3339
  }
3414
- if (showCaptionsRef.current && reviewModeRef.current) {
3415
- showCaption(text);
3340
+ if (!targetEl) {
3341
+ targetEl = resolveElementFromHints({
3342
+ fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
3343
+ testId: params.testId ?? fallbackHints?.testId,
3344
+ textContaining: params.textContaining ?? fallbackHints?.textContaining
3345
+ }, fallbackStep, tagStore);
3416
3346
  }
3417
- const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3418
- // Schedule narration immediately so the client keeps its speculative,
3419
- // low-latency behavior, but defer batch completion until playback settles.
3420
- waitForCompletion: false,
3421
- interrupt: action.params?.interrupt
3422
- });
3423
- return { result: text, settlePromise };
3424
- }
3425
- if (action.type === "play_timeline") {
3426
- const timelineShouldWait = await executeTimeline(action.params);
3427
- if (timelineShouldWait) shouldWait = true;
3428
- return { result: "timeline_executed" };
3347
+ return targetEl;
3429
3348
  }
3430
- if (action.type === "highlight_element") {
3431
- const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3432
- if (resolvedEl) {
3433
- if (isEditableWaitTarget(resolvedEl)) {
3434
- batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3349
+ });
3350
+ };
3351
+ const executeOne = async (action) => {
3352
+ assertNotInterrupted();
3353
+ console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
3354
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3355
+ const executeTimeline = async (params = {}) => {
3356
+ const segments = Array.isArray(params.segments) ? params.segments : [];
3357
+ for (let index = 0; index < segments.length; index += 1) {
3358
+ assertNotInterrupted();
3359
+ const segment = segments[index];
3360
+ const segmentText = (segment?.text ?? "").trim();
3361
+ const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
3362
+ const events = Array.isArray(segment?.events) ? segment.events : [];
3363
+ if (segmentDelayMs > 0) {
3364
+ await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
3365
+ }
3366
+ if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
3367
+ const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
3368
+ if (!gateTarget) {
3369
+ throw new Error(`timeline gate target not found for ${segment.gate.event}`);
3435
3370
  }
3436
- showHighlight(resolvedEl, action.params?.label || action.label);
3437
- resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3438
- return { result: "highlighted" };
3371
+ await waitForUserAction(gateTarget, segment.gate.event);
3439
3372
  }
3440
- if (!action.params?.optional) {
3441
- throw new Error(
3442
- `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3443
- );
3373
+ if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
3374
+ showCaption(segmentText);
3444
3375
  }
3445
- return { result: "highlight_optional_miss" };
3376
+ const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
3377
+ const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
3378
+ prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
3379
+ onNearEnd: nextSegmentText ? () => {
3380
+ void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
3381
+ } : void 0
3382
+ }) : Promise.resolve();
3383
+ const eventsPromise = (async () => {
3384
+ for (const event of events) {
3385
+ assertNotInterrupted();
3386
+ const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
3387
+ if (delayMs > 0) {
3388
+ await new Promise((resolve) => setTimeout(resolve, delayMs));
3389
+ }
3390
+ if (event?.action) {
3391
+ await executeOne(event.action);
3392
+ }
3393
+ }
3394
+ })();
3395
+ await Promise.all([speechPromise, eventsPromise]);
3446
3396
  }
3447
- if (action.type === "remove_highlight") {
3397
+ if (params.removeHighlightAtEnd !== false) {
3448
3398
  removeHighlight();
3449
- return { result: "highlight_removed" };
3450
3399
  }
3451
- if (action.type === "click_element") {
3452
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3453
- if (!targetEl) {
3454
- if (action.params?.optional) return { result: "click_optional_miss" };
3455
- throw new Error(
3456
- `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3457
- );
3458
- }
3459
- removeHighlight();
3460
- await performInteractiveClick(targetEl);
3461
- const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3462
- await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3463
- return { result: "clicked" };
3400
+ if (showCaptionsRef.current && reviewModeRef.current) {
3401
+ removeCaption();
3464
3402
  }
3465
- if (action.type === "fill_input") {
3466
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3467
- if (!targetEl) {
3468
- throw new Error(
3469
- `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3470
- );
3471
- }
3472
- const value = typeof action.params?.value === "string" ? action.params.value : "";
3473
- batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3474
- await performInteractiveFill(targetEl, value);
3475
- return { result: value };
3403
+ return !!params.waitForInput;
3404
+ };
3405
+ if (action.type === "speak") {
3406
+ const text = action.params?.text ?? "";
3407
+ if (!text.trim()) {
3408
+ return { result: null };
3476
3409
  }
3477
- if (action.type === "take_screenshot") {
3478
- const html2canvasModule = await import("html2canvas");
3479
- const html2canvas2 = html2canvasModule.default;
3480
- const canvas = await html2canvas2(document.body, {
3481
- useCORS: true,
3482
- allowTaint: true,
3483
- scale: Math.min(window.devicePixelRatio, 2),
3484
- width: window.innerWidth,
3485
- height: window.innerHeight,
3486
- x: window.scrollX,
3487
- y: window.scrollY,
3488
- logging: false
3489
- });
3490
- return { result: canvas.toDataURL("image/png") };
3410
+ if (showCaptionsRef.current && reviewModeRef.current) {
3411
+ showCaption(text);
3491
3412
  }
3492
- if (action.type === "navigate_to_url") {
3493
- const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3494
- if (!nextUrl) {
3495
- throw new Error("navigate_to_url missing url");
3413
+ const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
3414
+ // Schedule narration immediately so the client keeps its speculative,
3415
+ // low-latency behavior, but defer batch completion until playback settles.
3416
+ waitForCompletion: false,
3417
+ interrupt: action.params?.interrupt
3418
+ });
3419
+ return { result: text, settlePromise };
3420
+ }
3421
+ if (action.type === "play_timeline") {
3422
+ const timelineShouldWait = await executeTimeline(action.params);
3423
+ if (timelineShouldWait) shouldWait = true;
3424
+ return { result: "timeline_executed" };
3425
+ }
3426
+ if (action.type === "highlight_element") {
3427
+ const resolvedEl = await resolveTargetElement2(action.params, currentStep);
3428
+ if (resolvedEl) {
3429
+ if (isEditableWaitTarget(resolvedEl)) {
3430
+ batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
3496
3431
  }
3497
- await navigateToTourUrl(nextUrl);
3498
- const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3499
- await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3500
- return { result: nextUrl };
3432
+ showHighlight(resolvedEl, action.params?.label || action.label);
3433
+ resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
3434
+ return { result: "highlighted" };
3501
3435
  }
3502
- if (action.type === "execute_agent_action") {
3503
- const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3504
- if (!agentSocketId) {
3505
- throw new Error("No socketId available for execute_agent_action");
3506
- }
3507
- const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3508
- const res = await fetch(url, {
3509
- method: "POST",
3510
- headers: { "Content-Type": "application/json" },
3511
- body: JSON.stringify({
3512
- command: action.params?.command,
3513
- socketId: agentSocketId
3514
- })
3515
- });
3516
- if (!res.ok) {
3517
- throw new Error(`execute_agent_action failed: ${await res.text()}`);
3518
- }
3519
- if (action.params?.wait !== false) {
3520
- await res.json();
3521
- }
3522
- const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3523
- await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
3524
- await syncAOM();
3525
- return { result: action.params?.command ?? "executed" };
3436
+ if (!action.params?.optional) {
3437
+ throw new Error(
3438
+ `highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3439
+ );
3526
3440
  }
3527
- if (action.type === "wait_for_user_action") {
3528
- const targetEl = await resolveTargetElement2(action.params, currentStep);
3529
- if (!targetEl) {
3530
- throw new Error("wait_for_user_action target not found");
3531
- }
3532
- const eventName = action.params?.event ?? "click";
3533
- await waitForUserAction(targetEl, eventName);
3534
- return { result: `waited_for_${eventName}` };
3441
+ return { result: "highlight_optional_miss" };
3442
+ }
3443
+ if (action.type === "remove_highlight") {
3444
+ removeHighlight();
3445
+ return { result: "highlight_removed" };
3446
+ }
3447
+ if (action.type === "click_element") {
3448
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3449
+ if (!targetEl) {
3450
+ if (action.params?.optional) return { result: "click_optional_miss" };
3451
+ throw new Error(
3452
+ `click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3453
+ );
3535
3454
  }
3536
- if (action.type === "wait_for_input") {
3537
- shouldWait = true;
3538
- return { result: "waiting_for_input" };
3455
+ removeHighlight();
3456
+ await performInteractiveClick(targetEl);
3457
+ const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3458
+ await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
3459
+ return { result: "clicked" };
3460
+ }
3461
+ if (action.type === "fill_input") {
3462
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3463
+ if (!targetEl) {
3464
+ throw new Error(
3465
+ `fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
3466
+ );
3539
3467
  }
3540
- if (action.type === "end_tour") {
3541
- handleTourEnd();
3542
- return { result: "ended" };
3468
+ const value = typeof action.params?.value === "string" ? action.params.value : "";
3469
+ batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
3470
+ await performInteractiveFill(targetEl, value);
3471
+ return { result: value };
3472
+ }
3473
+ if (action.type === "take_screenshot") {
3474
+ const html2canvasModule = await import("html2canvas");
3475
+ const html2canvas2 = html2canvasModule.default;
3476
+ const canvas = await html2canvas2(document.body, {
3477
+ useCORS: true,
3478
+ allowTaint: true,
3479
+ scale: Math.min(window.devicePixelRatio, 2),
3480
+ width: window.innerWidth,
3481
+ height: window.innerHeight,
3482
+ x: window.scrollX,
3483
+ y: window.scrollY,
3484
+ logging: false
3485
+ });
3486
+ return { result: canvas.toDataURL("image/png") };
3487
+ }
3488
+ if (action.type === "navigate_to_url") {
3489
+ const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
3490
+ if (!nextUrl) {
3491
+ throw new Error("navigate_to_url missing url");
3543
3492
  }
3544
- console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3545
- return { result: "unknown_action_skipped" };
3546
- };
3547
- try {
3548
- const resultsBuffer = new Array(payload.commands.length);
3549
- const pendingUIActions = [];
3550
- for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3551
- const command = payload.commands[commandIndex];
3552
- assertNotInterrupted();
3553
- const isTerminal = isTerminalAction(command);
3554
- if (isTerminal) {
3555
- await Promise.all(pendingUIActions);
3556
- pendingUIActions.length = 0;
3557
- }
3558
- const executionPromise = (async () => {
3559
- const execution = await executeOne(command);
3560
- await execution.settlePromise;
3561
- resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3562
- })();
3563
- if (isTerminal) {
3564
- await executionPromise;
3565
- } else {
3566
- pendingUIActions.push(executionPromise);
3567
- }
3493
+ await navigateToTourUrl(nextUrl);
3494
+ const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3495
+ await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
3496
+ return { result: nextUrl };
3497
+ }
3498
+ if (action.type === "execute_agent_action") {
3499
+ const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
3500
+ if (!agentSocketId) {
3501
+ throw new Error("No socketId available for execute_agent_action");
3568
3502
  }
3569
- await Promise.all(pendingUIActions);
3570
- resultsBuffer.forEach((res) => {
3571
- if (res) results.push(res);
3503
+ const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
3504
+ const res = await fetch(url, {
3505
+ method: "POST",
3506
+ headers: { "Content-Type": "application/json" },
3507
+ body: JSON.stringify({
3508
+ command: action.params?.command,
3509
+ socketId: agentSocketId
3510
+ })
3572
3511
  });
3573
- await syncAOM();
3574
- } catch (err) {
3575
- commandInFlightRef.current = false;
3576
- const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3577
- if (interrupted) {
3578
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3579
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3580
- stepOrder: stepIndexRef.current,
3581
- eventType: "command_batch_interrupted",
3582
- payload: {
3583
- commandBatchId,
3584
- partialResults: results
3585
- },
3586
- currentStepOrder: stepIndexRef.current
3587
- });
3588
- }
3589
- emitIfOpen("tour:action_result", {
3590
- success: true,
3591
- interrupted: true,
3592
- results,
3593
- commandBatchId,
3594
- runId: runIdRef.current,
3595
- turnId: turnIdRef.current
3596
- });
3597
- clearCommandBatchId();
3598
- return;
3512
+ if (!res.ok) {
3513
+ throw new Error(`execute_agent_action failed: ${await res.text()}`);
3599
3514
  }
3600
- console.error("[TourClient] Command batch execution failed:", err);
3601
- if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3602
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3603
- stepOrder: stepIndexRef.current,
3604
- eventType: "command_batch_failed",
3605
- payload: {
3606
- commandBatchId,
3607
- error: String(err),
3608
- partialResults: results
3609
- },
3610
- status: "active",
3611
- currentStepOrder: stepIndexRef.current
3612
- });
3515
+ if (action.params?.wait !== false) {
3516
+ await res.json();
3613
3517
  }
3614
- emitIfOpen("tour:action_result", {
3615
- success: false,
3616
- reason: "execution_error",
3617
- error: String(err),
3618
- results,
3619
- commandBatchId,
3620
- runId: runIdRef.current,
3621
- turnId: turnIdRef.current
3622
- });
3623
- clearCommandBatchId();
3624
- return;
3518
+ const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3519
+ await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
3520
+ await syncAOM();
3521
+ return { result: action.params?.command ?? "executed" };
3625
3522
  }
3626
- commandInFlightRef.current = false;
3627
- if (shouldWait && !skipRequestedRef.current) {
3628
- const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3629
- const waitCondition = currentStep?.onboarding?.waitCondition;
3630
- const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3631
- const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3632
- const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3633
- let manualWaitPromise = null;
3634
- let manualWaitKind = null;
3635
- const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3636
- const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3637
- runCleanup(pendingManualWaitCleanupRef.current);
3638
- pendingManualWaitCleanupRef.current = null;
3639
- if (waitTargetHints) {
3640
- let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3641
- if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3642
- manualWaitTarget = preferredWaitTarget;
3643
- console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3644
- }
3645
- if (manualWaitTarget) {
3646
- const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3647
- manualWaitPromise = manualWait.promise;
3648
- manualWaitKind = manualWait.kind;
3649
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3650
- }
3523
+ if (action.type === "wait_for_user_action") {
3524
+ const targetEl = await resolveTargetElement2(action.params, currentStep);
3525
+ if (!targetEl) {
3526
+ throw new Error("wait_for_user_action target not found");
3651
3527
  }
3652
- if (!manualWaitPromise && preferredWaitTarget) {
3653
- const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3654
- manualWaitPromise = manualWait.promise;
3655
- manualWaitKind = manualWait.kind;
3656
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3657
- console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3528
+ const eventName = action.params?.event ?? "click";
3529
+ await waitForUserAction(targetEl, eventName);
3530
+ return { result: `waited_for_${eventName}` };
3531
+ }
3532
+ if (action.type === "wait_for_input") {
3533
+ shouldWait = true;
3534
+ return { result: "waiting_for_input" };
3535
+ }
3536
+ if (action.type === "end_tour") {
3537
+ handleTourEnd();
3538
+ return { result: "ended" };
3539
+ }
3540
+ console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
3541
+ return { result: "unknown_action_skipped" };
3542
+ };
3543
+ try {
3544
+ const resultsBuffer = new Array(payload.commands.length);
3545
+ const pendingUIActions = [];
3546
+ for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
3547
+ const command = payload.commands[commandIndex];
3548
+ assertNotInterrupted();
3549
+ const isTerminal = isTerminalAction(command);
3550
+ if (isTerminal) {
3551
+ await Promise.all(pendingUIActions);
3552
+ pendingUIActions.length = 0;
3658
3553
  }
3659
- if (!manualWaitPromise && inputLikeWait) {
3660
- const firstInput = document.querySelector(
3661
- 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3662
- );
3663
- if (firstInput) {
3664
- const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3665
- manualWaitPromise = manualWait.promise;
3666
- manualWaitKind = manualWait.kind;
3667
- pendingManualWaitCleanupRef.current = manualWait.cleanup;
3668
- console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3669
- }
3554
+ const executionPromise = (async () => {
3555
+ const execution = await executeOne(command);
3556
+ await execution.settlePromise;
3557
+ resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
3558
+ })();
3559
+ if (isTerminal) {
3560
+ await executionPromise;
3561
+ } else {
3562
+ pendingUIActions.push(executionPromise);
3670
3563
  }
3671
- setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3564
+ }
3565
+ await Promise.all(pendingUIActions);
3566
+ resultsBuffer.forEach((res) => {
3567
+ if (res) results.push(res);
3568
+ });
3569
+ await syncAOM();
3570
+ } catch (err) {
3571
+ commandInFlightRef.current = false;
3572
+ const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
3573
+ if (interrupted) {
3672
3574
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3673
3575
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3674
3576
  stepOrder: stepIndexRef.current,
3675
- eventType: "waiting_for_input",
3577
+ eventType: "command_batch_interrupted",
3676
3578
  payload: {
3677
3579
  commandBatchId,
3678
- results
3580
+ partialResults: results
3679
3581
  },
3680
3582
  currentStepOrder: stepIndexRef.current
3681
3583
  });
3682
3584
  }
3683
3585
  emitIfOpen("tour:action_result", {
3684
3586
  success: true,
3685
- waitingForInput: true,
3587
+ interrupted: true,
3686
3588
  results,
3687
3589
  commandBatchId,
3688
3590
  runId: runIdRef.current,
3689
3591
  turnId: turnIdRef.current
3690
3592
  });
3691
3593
  clearCommandBatchId();
3692
- const voiceOrTextWaitPromise = new Promise((resolve) => {
3693
- if (pendingInputBufRef.current) {
3694
- const flushed = pendingInputBufRef.current;
3695
- pendingInputBufRef.current = null;
3696
- resolve(flushed);
3697
- return;
3698
- }
3699
- voiceInputResolveRef.current = (text) => {
3700
- voiceInputResolveRef.current = null;
3701
- resolve(text);
3702
- };
3703
- });
3704
- Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3705
- runCleanup(pendingManualWaitCleanupRef.current);
3706
- pendingManualWaitCleanupRef.current = null;
3707
- voiceInputResolveRef.current = null;
3708
- setPlaybackState("executing");
3709
- const transcript = userText.trim();
3710
- if (!transcript) {
3711
- return;
3712
- }
3713
- const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3714
- await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
3715
- await syncAOM();
3716
- emitIfOpen("tour:user_input", {
3717
- transcript,
3718
- runId: runIdRef.current,
3719
- turnId: turnIdRef.current
3720
- });
3721
- });
3722
3594
  return;
3723
3595
  }
3596
+ console.error("[TourClient] Command batch execution failed:", err);
3724
3597
  if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3725
3598
  void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3726
3599
  stepOrder: stepIndexRef.current,
3727
- eventType: "command_batch_completed",
3600
+ eventType: "command_batch_failed",
3728
3601
  payload: {
3729
3602
  commandBatchId,
3730
- results
3603
+ error: String(err),
3604
+ partialResults: results
3731
3605
  },
3606
+ status: "active",
3732
3607
  currentStepOrder: stepIndexRef.current
3733
3608
  });
3734
3609
  }
3735
3610
  emitIfOpen("tour:action_result", {
3736
- success: true,
3611
+ success: false,
3612
+ reason: "execution_error",
3613
+ error: String(err),
3737
3614
  results,
3738
3615
  commandBatchId,
3739
3616
  runId: runIdRef.current,
3740
3617
  turnId: turnIdRef.current
3741
3618
  });
3742
3619
  clearCommandBatchId();
3743
- });
3744
- socket.on("tour:start", async (tourData) => {
3745
- if (isActiveRef.current) return;
3746
- runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3747
- const tour = tourData.tourContext ?? tourRef.current;
3748
- const expType = experienceTypeRef.current;
3749
- if (tour?.type && tour.type !== expType) {
3750
- console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3751
- return;
3620
+ return;
3621
+ }
3622
+ commandInFlightRef.current = false;
3623
+ if (shouldWait && !skipRequestedRef.current) {
3624
+ const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
3625
+ const waitCondition = currentStep?.onboarding?.waitCondition;
3626
+ const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
3627
+ const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
3628
+ const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
3629
+ let manualWaitPromise = null;
3630
+ let manualWaitKind = null;
3631
+ const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
3632
+ const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
3633
+ runCleanup(pendingManualWaitCleanupRef.current);
3634
+ pendingManualWaitCleanupRef.current = null;
3635
+ if (waitTargetHints) {
3636
+ let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
3637
+ if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
3638
+ manualWaitTarget = preferredWaitTarget;
3639
+ console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
3640
+ }
3641
+ if (manualWaitTarget) {
3642
+ const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
3643
+ manualWaitPromise = manualWait.promise;
3644
+ manualWaitKind = manualWait.kind;
3645
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3646
+ }
3752
3647
  }
3753
- skipRequestedRef.current = false;
3754
- const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3755
- isActiveRef.current = true;
3756
- setIsActive(true);
3757
- setActiveTour(tour ?? null);
3758
- tourRef.current = tour ?? null;
3759
- setTotalSteps(total);
3760
- stepIndexRef.current = 0;
3761
- setCurrentStepIndex(0);
3762
- setPlaybackState("intro");
3763
- if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3764
- void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3765
- stepOrder: 0,
3766
- eventType: "tour_started",
3648
+ if (!manualWaitPromise && preferredWaitTarget) {
3649
+ const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
3650
+ manualWaitPromise = manualWait.promise;
3651
+ manualWaitKind = manualWait.kind;
3652
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3653
+ console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
3654
+ }
3655
+ if (!manualWaitPromise && inputLikeWait) {
3656
+ const firstInput = document.querySelector(
3657
+ 'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
3658
+ );
3659
+ if (firstInput) {
3660
+ const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
3661
+ manualWaitPromise = manualWait.promise;
3662
+ manualWaitKind = manualWait.kind;
3663
+ pendingManualWaitCleanupRef.current = manualWait.cleanup;
3664
+ console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
3665
+ }
3666
+ }
3667
+ setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
3668
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3669
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3670
+ stepOrder: stepIndexRef.current,
3671
+ eventType: "waiting_for_input",
3767
3672
  payload: {
3768
- totalSteps: total,
3769
- source: "sdk_test_preview"
3673
+ commandBatchId,
3674
+ results
3770
3675
  },
3771
- currentStepOrder: 0
3676
+ currentStepOrder: stepIndexRef.current
3772
3677
  });
3773
3678
  }
3774
- try {
3775
- const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3776
- const aom = generateMinifiedAOM2();
3777
- if (socketRef.current === socket && socket.connected) {
3778
- socket.emit("tour:sync_dom", {
3779
- url: window.location.pathname + window.location.search + window.location.hash,
3780
- aom: aom.nodes,
3781
- domSummary: captureDomSummary()
3782
- });
3679
+ emitIfOpen("tour:action_result", {
3680
+ success: true,
3681
+ waitingForInput: true,
3682
+ results,
3683
+ commandBatchId,
3684
+ runId: runIdRef.current,
3685
+ turnId: turnIdRef.current
3686
+ });
3687
+ clearCommandBatchId();
3688
+ const voiceOrTextWaitPromise = new Promise((resolve) => {
3689
+ if (pendingInputBufRef.current) {
3690
+ const flushed = pendingInputBufRef.current;
3691
+ pendingInputBufRef.current = null;
3692
+ resolve(flushed);
3693
+ return;
3783
3694
  }
3784
- } catch (e) {
3785
- console.warn("[TourClient] Initial DOM sync failed:", e);
3786
- }
3695
+ voiceInputResolveRef.current = (text) => {
3696
+ voiceInputResolveRef.current = null;
3697
+ resolve(text);
3698
+ };
3699
+ });
3700
+ Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
3701
+ runCleanup(pendingManualWaitCleanupRef.current);
3702
+ pendingManualWaitCleanupRef.current = null;
3703
+ voiceInputResolveRef.current = null;
3704
+ setPlaybackState("executing");
3705
+ const transcript = userText.trim();
3706
+ if (!transcript) {
3707
+ return;
3708
+ }
3709
+ const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
3710
+ await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
3711
+ await syncAOM();
3712
+ emitIfOpen("tour:user_input", {
3713
+ transcript,
3714
+ runId: runIdRef.current,
3715
+ turnId: turnIdRef.current
3716
+ });
3717
+ });
3718
+ return;
3719
+ }
3720
+ if (reviewModeRef.current && activeTourId && activePreviewRunId) {
3721
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
3722
+ stepOrder: stepIndexRef.current,
3723
+ eventType: "command_batch_completed",
3724
+ payload: {
3725
+ commandBatchId,
3726
+ results
3727
+ },
3728
+ currentStepOrder: stepIndexRef.current
3729
+ });
3730
+ }
3731
+ emitIfOpen("tour:action_result", {
3732
+ success: true,
3733
+ results,
3734
+ commandBatchId,
3735
+ runId: runIdRef.current,
3736
+ turnId: turnIdRef.current
3787
3737
  });
3788
- socket.on("tour:update", (payload) => {
3789
- const updatedTour = payload?.tourContext;
3790
- if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3791
- return;
3738
+ clearCommandBatchId();
3739
+ });
3740
+ socket.on("tour:start", async (tourData) => {
3741
+ if (isActiveRef.current) return;
3742
+ runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
3743
+ const tour = tourData.tourContext ?? tourRef.current;
3744
+ const expType = experienceTypeRef.current;
3745
+ if (tour?.type && tour.type !== expType) {
3746
+ console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
3747
+ return;
3748
+ }
3749
+ skipRequestedRef.current = false;
3750
+ const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
3751
+ isActiveRef.current = true;
3752
+ setIsActive(true);
3753
+ setActiveTour(tour ?? null);
3754
+ tourRef.current = tour ?? null;
3755
+ setTotalSteps(total);
3756
+ stepIndexRef.current = 0;
3757
+ setCurrentStepIndex(0);
3758
+ setPlaybackState("intro");
3759
+ if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
3760
+ void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
3761
+ stepOrder: 0,
3762
+ eventType: "tour_started",
3763
+ payload: {
3764
+ totalSteps: total,
3765
+ source: "sdk_test_preview"
3766
+ },
3767
+ currentStepOrder: 0
3768
+ });
3769
+ }
3770
+ try {
3771
+ const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3772
+ const aom = generateMinifiedAOM2();
3773
+ if (socketRef.current === socket && socket.connected) {
3774
+ socket.emit("tour:sync_dom", {
3775
+ url: window.location.pathname + window.location.search + window.location.hash,
3776
+ aom: aom.nodes,
3777
+ domSummary: captureDomSummary()
3778
+ });
3792
3779
  }
3793
- tourRef.current = updatedTour;
3794
- setActiveTour(updatedTour);
3795
- const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3796
- setTotalSteps(nextTotal);
3797
- if (typeof payload.currentStepIndex === "number") {
3798
- const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3799
- stepIndexRef.current = clampedStepIndex;
3800
- setCurrentStepIndex(clampedStepIndex);
3801
- if (nextTotal > 0) {
3802
- onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3803
- }
3780
+ } catch (e) {
3781
+ console.warn("[TourClient] Initial DOM sync failed:", e);
3782
+ }
3783
+ });
3784
+ socket.on("tour:update", (payload) => {
3785
+ const updatedTour = payload?.tourContext;
3786
+ if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
3787
+ return;
3788
+ }
3789
+ tourRef.current = updatedTour;
3790
+ setActiveTour(updatedTour);
3791
+ const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
3792
+ setTotalSteps(nextTotal);
3793
+ if (typeof payload.currentStepIndex === "number") {
3794
+ const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
3795
+ stepIndexRef.current = clampedStepIndex;
3796
+ setCurrentStepIndex(clampedStepIndex);
3797
+ if (nextTotal > 0) {
3798
+ onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
3804
3799
  }
3805
- });
3806
- socket.on("tour:end", () => {
3807
- setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3808
- handleTourEnd();
3809
- });
3810
- socket.on("tour:debug_log", (entry) => {
3811
- const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3812
- if (isDev) {
3813
- console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3814
- if (typeof window !== "undefined") {
3815
- window.dispatchEvent(new CustomEvent("modelnex-debug", {
3816
- detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3817
- }));
3818
- }
3800
+ }
3801
+ });
3802
+ socket.on("tour:end", () => {
3803
+ setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
3804
+ handleTourEnd();
3805
+ });
3806
+ socket.on("tour:debug_log", (entry) => {
3807
+ const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
3808
+ if (isDev) {
3809
+ console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
3810
+ if (typeof window !== "undefined") {
3811
+ window.dispatchEvent(new CustomEvent("modelnex-debug", {
3812
+ detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
3813
+ }));
3819
3814
  }
3820
- });
3821
- console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3815
+ }
3822
3816
  });
3817
+ console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
3823
3818
  return () => {
3824
- cancelled = true;
3825
3819
  const toClose = createdSocket ?? socketRef.current;
3826
3820
  if (toClose) {
3827
3821
  toClose.disconnect();
@@ -3838,7 +3832,12 @@ function useTourPlayback({
3838
3832
  const s = socketRef.current;
3839
3833
  const profile = userProfile;
3840
3834
  if (!s?.connected || !websiteId || !profile?.userId) return;
3841
- s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3835
+ const timer = setTimeout(() => {
3836
+ if (s.connected) {
3837
+ s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
3838
+ }
3839
+ }, 150);
3840
+ return () => clearTimeout(timer);
3842
3841
  }, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
3843
3842
  (0, import_react12.useEffect)(() => {
3844
3843
  if (!showCaptions || !isReviewMode) {
@@ -3863,11 +3862,13 @@ function useTourPlayback({
3863
3862
  if (!socketRef.current?.connected || !isActiveRef.current) return;
3864
3863
  const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
3865
3864
  const aom = generateMinifiedAOM2();
3866
- socketRef.current.emit("tour:sync_dom", {
3867
- url: window.location.pathname + window.location.search + window.location.hash,
3868
- aom: aom.nodes,
3869
- domSummary: captureDomSummary()
3870
- });
3865
+ if (socketRef.current?.connected) {
3866
+ socketRef.current.emit("tour:sync_dom", {
3867
+ url: window.location.pathname + window.location.search + window.location.hash,
3868
+ aom: aom.nodes,
3869
+ domSummary: captureDomSummary()
3870
+ });
3871
+ }
3871
3872
  }, []);
3872
3873
  const interruptExecution = (0, import_react12.useCallback)((transcript) => {
3873
3874
  if (!socketRef.current?.connected || !isActiveRef.current) return false;
@@ -3878,21 +3879,23 @@ function useTourPlayback({
3878
3879
  removeHighlight();
3879
3880
  removeCaption();
3880
3881
  voice.stopSpeaking();
3881
- socketRef.current.emit("tour:action_result", {
3882
- success: true,
3883
- interrupted: true,
3884
- transcript,
3885
- commandBatchId: activeCommandBatchIdRef.current,
3886
- runId: runIdRef.current,
3887
- turnId: turnIdRef.current
3888
- });
3889
- activeCommandBatchIdRef.current = null;
3890
- socketRef.current.emit("tour:user_input", {
3891
- transcript,
3892
- interrupted: true,
3893
- runId: runIdRef.current,
3894
- turnId: turnIdRef.current
3895
- });
3882
+ if (socketRef.current?.connected) {
3883
+ socketRef.current.emit("tour:action_result", {
3884
+ success: true,
3885
+ interrupted: true,
3886
+ transcript,
3887
+ commandBatchId: activeCommandBatchIdRef.current,
3888
+ runId: runIdRef.current,
3889
+ turnId: turnIdRef.current
3890
+ });
3891
+ activeCommandBatchIdRef.current = null;
3892
+ socketRef.current.emit("tour:user_input", {
3893
+ transcript,
3894
+ interrupted: true,
3895
+ runId: runIdRef.current,
3896
+ turnId: turnIdRef.current
3897
+ });
3898
+ }
3896
3899
  setPlaybackState("thinking");
3897
3900
  return true;
3898
3901
  }, [voice]);
@@ -4010,11 +4013,13 @@ function useTourPlayback({
4010
4013
  previewRunIdRef.current = null;
4011
4014
  }
4012
4015
  tourRef.current = tour;
4013
- socketRef.current.emit("tour:request_start", {
4014
- tourId: tour.id,
4015
- previewRunId: previewRunIdRef.current,
4016
- tourContext: tour
4017
- });
4016
+ if (socketRef.current?.connected) {
4017
+ socketRef.current.emit("tour:request_start", {
4018
+ tourId: tour.id,
4019
+ previewRunId: previewRunIdRef.current,
4020
+ tourContext: tour
4021
+ });
4022
+ }
4018
4023
  }, [serverUrl, websiteId]);
4019
4024
  (0, import_react12.useEffect)(() => {
4020
4025
  if (!enableAutoDiscovery) return;
@@ -5462,7 +5467,7 @@ function useVoice(serverUrl) {
5462
5467
  (async () => {
5463
5468
  try {
5464
5469
  const ioModule = await import("socket.io-client");
5465
- const io2 = ioModule.default || ioModule;
5470
+ const io3 = ioModule.default || ioModule;
5466
5471
  const stream = await navigator.mediaDevices.getUserMedia({
5467
5472
  audio: {
5468
5473
  echoCancellation: true,
@@ -5484,7 +5489,7 @@ function useVoice(serverUrl) {
5484
5489
  audioBitsPerSecond: 128e3
5485
5490
  });
5486
5491
  mediaRecorderRef.current = recorder;
5487
- const socket = io2(serverUrl, {
5492
+ const socket = io3(serverUrl, {
5488
5493
  path: "/socket.io",
5489
5494
  transports: resolveSocketIoTransports(serverUrl, "websocket-first")
5490
5495
  });
@@ -11005,7 +11010,7 @@ var ModelNexProvider = ({
11005
11010
  socketId,
11006
11011
  devMode
11007
11012
  }),
11008
- [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
11013
+ [serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
11009
11014
  );
11010
11015
  return import_react21.default.createElement(
11011
11016
  ModelNexContext.Provider,