@modelnex/sdk 0.5.15 → 0.5.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +647 -576
- package/dist/index.mjs +647 -576
- package/package.json +11 -12
- package/dist/aom-GA6W42DG.mjs +0 -71
- package/dist/aom-J6NYMGDW.mjs +0 -69
- package/dist/chunk-6DZX6EAA.mjs +0 -37
- package/dist/dom-sync-Y7Z7TOU6.mjs +0 -57
package/dist/index.js
CHANGED
|
@@ -2648,6 +2648,76 @@ async function retryLookup({
|
|
|
2648
2648
|
}
|
|
2649
2649
|
}
|
|
2650
2650
|
|
|
2651
|
+
// src/utils/tour-socket-pool.ts
|
|
2652
|
+
var import_socket2 = require("socket.io-client");
|
|
2653
|
+
function isSocketWritable(socket) {
|
|
2654
|
+
if (!socket?.connected) return false;
|
|
2655
|
+
const engine = socket.io?.engine;
|
|
2656
|
+
if (!engine) return true;
|
|
2657
|
+
if (typeof engine.readyState === "string" && engine.readyState !== "open") return false;
|
|
2658
|
+
if (engine.transport && "writable" in engine.transport && engine.transport.writable === false) return false;
|
|
2659
|
+
return true;
|
|
2660
|
+
}
|
|
2661
|
+
function emitSocketEvent(socket, event, payload) {
|
|
2662
|
+
if (!isSocketWritable(socket)) return false;
|
|
2663
|
+
socket.emit(event, payload);
|
|
2664
|
+
return true;
|
|
2665
|
+
}
|
|
2666
|
+
function createTourSocketPool({
|
|
2667
|
+
createSocket = (serverUrl) => (0, import_socket2.io)(serverUrl, {
|
|
2668
|
+
path: "/socket.io",
|
|
2669
|
+
transports: resolveSocketIoTransports(serverUrl, "polling-first"),
|
|
2670
|
+
autoConnect: true,
|
|
2671
|
+
reconnection: true,
|
|
2672
|
+
reconnectionAttempts: 10,
|
|
2673
|
+
reconnectionDelay: 1e3
|
|
2674
|
+
}),
|
|
2675
|
+
releaseDelayMs = 2500,
|
|
2676
|
+
scheduleRelease = (callback, delayMs) => setTimeout(callback, delayMs),
|
|
2677
|
+
cancelRelease = (timer) => clearTimeout(timer)
|
|
2678
|
+
} = {}) {
|
|
2679
|
+
const pooledSockets = /* @__PURE__ */ new Map();
|
|
2680
|
+
return {
|
|
2681
|
+
acquire(serverUrl) {
|
|
2682
|
+
const pooled = pooledSockets.get(serverUrl);
|
|
2683
|
+
if (pooled) {
|
|
2684
|
+
if (pooled.releaseTimer) {
|
|
2685
|
+
cancelRelease(pooled.releaseTimer);
|
|
2686
|
+
pooled.releaseTimer = null;
|
|
2687
|
+
}
|
|
2688
|
+
pooled.leases += 1;
|
|
2689
|
+
return pooled.socket;
|
|
2690
|
+
}
|
|
2691
|
+
const socket = createSocket(serverUrl);
|
|
2692
|
+
pooledSockets.set(serverUrl, {
|
|
2693
|
+
socket,
|
|
2694
|
+
leases: 1,
|
|
2695
|
+
releaseTimer: null
|
|
2696
|
+
});
|
|
2697
|
+
return socket;
|
|
2698
|
+
},
|
|
2699
|
+
release(serverUrl, socket) {
|
|
2700
|
+
const pooled = pooledSockets.get(serverUrl);
|
|
2701
|
+
if (!pooled || pooled.socket !== socket) return;
|
|
2702
|
+
pooled.leases = Math.max(0, pooled.leases - 1);
|
|
2703
|
+
if (pooled.leases > 0) return;
|
|
2704
|
+
pooled.releaseTimer = scheduleRelease(() => {
|
|
2705
|
+
const latest = pooledSockets.get(serverUrl);
|
|
2706
|
+
if (!latest || latest !== pooled || latest.leases > 0) return;
|
|
2707
|
+
latest.socket.disconnect();
|
|
2708
|
+
pooledSockets.delete(serverUrl);
|
|
2709
|
+
}, releaseDelayMs);
|
|
2710
|
+
},
|
|
2711
|
+
// Test-only introspection
|
|
2712
|
+
getSnapshot(serverUrl) {
|
|
2713
|
+
const pooled = pooledSockets.get(serverUrl);
|
|
2714
|
+
if (!pooled) return null;
|
|
2715
|
+
return { leases: pooled.leases, socket: pooled.socket };
|
|
2716
|
+
}
|
|
2717
|
+
};
|
|
2718
|
+
}
|
|
2719
|
+
var tourSocketPool = createTourSocketPool();
|
|
2720
|
+
|
|
2651
2721
|
// src/hooks/useTourPlayback.ts
|
|
2652
2722
|
function resolveElement(step) {
|
|
2653
2723
|
const el = step.element;
|
|
@@ -3184,11 +3254,13 @@ function useTourPlayback({
|
|
|
3184
3254
|
const socketRef = (0, import_react12.useRef)(null);
|
|
3185
3255
|
const socketIdRef = (0, import_react12.useRef)(socketId);
|
|
3186
3256
|
const commandUrlRef = (0, import_react12.useRef)(commandUrl);
|
|
3257
|
+
const websiteIdRef = (0, import_react12.useRef)(websiteId);
|
|
3187
3258
|
const onStepChangeRef = (0, import_react12.useRef)(onStepChange);
|
|
3188
3259
|
const isActiveRef = (0, import_react12.useRef)(false);
|
|
3189
3260
|
const activeCommandBatchIdRef = (0, import_react12.useRef)(null);
|
|
3190
3261
|
socketIdRef.current = socketId;
|
|
3191
3262
|
commandUrlRef.current = commandUrl;
|
|
3263
|
+
websiteIdRef.current = websiteId;
|
|
3192
3264
|
onStepChangeRef.current = onStepChange;
|
|
3193
3265
|
isActiveRef.current = isActive;
|
|
3194
3266
|
reviewModeRef.current = isReviewMode;
|
|
@@ -3199,646 +3271,646 @@ function useTourPlayback({
|
|
|
3199
3271
|
(0, import_react12.useEffect)(() => {
|
|
3200
3272
|
if (disabled) return;
|
|
3201
3273
|
if (typeof window === "undefined") return;
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
const
|
|
3207
|
-
const
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
transports: resolveSocketIoTransports(serverUrl, "polling-first")
|
|
3211
|
-
});
|
|
3212
|
-
if (cancelled) {
|
|
3213
|
-
socket.disconnect();
|
|
3214
|
-
return;
|
|
3274
|
+
const socket = tourSocketPool.acquire(serverUrl);
|
|
3275
|
+
socketRef.current = socket;
|
|
3276
|
+
const handleConnect = () => {
|
|
3277
|
+
console.log("[TourClient] Connected to tour agent server:", socket.id);
|
|
3278
|
+
const profile = userProfileRef.current;
|
|
3279
|
+
const currentWebsiteId = websiteIdRef.current;
|
|
3280
|
+
if (currentWebsiteId && profile?.userId) {
|
|
3281
|
+
emitSocketEvent(socket, "tour:init", { websiteId: currentWebsiteId, userId: profile.userId, userType: profile.type });
|
|
3215
3282
|
}
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3283
|
+
};
|
|
3284
|
+
const handleServerState = (payload) => {
|
|
3285
|
+
if (typeof payload?.runId === "number") {
|
|
3286
|
+
runIdRef.current = payload.runId;
|
|
3287
|
+
}
|
|
3288
|
+
if (typeof payload?.turnId === "string" || payload?.turnId === null) {
|
|
3289
|
+
turnIdRef.current = payload.turnId ?? null;
|
|
3290
|
+
}
|
|
3291
|
+
setServerState(payload);
|
|
3292
|
+
};
|
|
3293
|
+
const handleCommandCancel = (payload) => {
|
|
3294
|
+
console.log("[TourClient] Received command_cancel:", payload);
|
|
3295
|
+
if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
|
|
3296
|
+
activeCommandBatchIdRef.current = null;
|
|
3297
|
+
activeExecutionTokenRef.current++;
|
|
3298
|
+
commandInFlightRef.current = false;
|
|
3299
|
+
setPlaybackState("idle");
|
|
3300
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3301
|
+
window.speechSynthesis.cancel();
|
|
3231
3302
|
}
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
if (
|
|
3303
|
+
}
|
|
3304
|
+
};
|
|
3305
|
+
const handleCommand = async (payload) => {
|
|
3306
|
+
const emitIfOpen = (ev, data) => {
|
|
3307
|
+
if (socketRef.current !== socket) return;
|
|
3308
|
+
emitSocketEvent(socket, ev, data);
|
|
3309
|
+
};
|
|
3310
|
+
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3311
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3312
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3313
|
+
if (voiceInputResolveRef.current) {
|
|
3314
|
+
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3315
|
+
voiceInputResolveRef.current = null;
|
|
3316
|
+
resolvePendingVoiceInput("");
|
|
3317
|
+
}
|
|
3318
|
+
setPlaybackState("executing");
|
|
3319
|
+
commandInFlightRef.current = true;
|
|
3320
|
+
const commandBatchId = payload.commandBatchId ?? null;
|
|
3321
|
+
turnIdRef.current = payload.turnId ?? turnIdRef.current;
|
|
3322
|
+
const clearCommandBatchId = () => {
|
|
3323
|
+
if (activeCommandBatchIdRef.current === commandBatchId) {
|
|
3237
3324
|
activeCommandBatchIdRef.current = null;
|
|
3238
|
-
activeExecutionTokenRef.current++;
|
|
3239
|
-
commandInFlightRef.current = false;
|
|
3240
|
-
setPlaybackState("idle");
|
|
3241
|
-
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3242
|
-
window.speechSynthesis.cancel();
|
|
3243
|
-
}
|
|
3244
|
-
}
|
|
3245
|
-
});
|
|
3246
|
-
socket.on("tour:command", async (payload) => {
|
|
3247
|
-
const emitIfOpen = (ev, data) => {
|
|
3248
|
-
if (socketRef.current !== socket) return;
|
|
3249
|
-
if (!socket.connected) return;
|
|
3250
|
-
socket.emit(ev, data);
|
|
3251
|
-
};
|
|
3252
|
-
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3253
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3254
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3255
|
-
if (voiceInputResolveRef.current) {
|
|
3256
|
-
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3257
|
-
voiceInputResolveRef.current = null;
|
|
3258
|
-
resolvePendingVoiceInput("");
|
|
3259
3325
|
}
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3326
|
+
};
|
|
3327
|
+
activeCommandBatchIdRef.current = commandBatchId;
|
|
3328
|
+
const executionToken = ++activeExecutionTokenRef.current;
|
|
3329
|
+
const activeTourId = tourRef.current?.id;
|
|
3330
|
+
const activePreviewRunId = previewRunIdRef.current;
|
|
3331
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3332
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3333
|
+
stepOrder: payload.stepIndex,
|
|
3334
|
+
eventType: "command_batch_received",
|
|
3335
|
+
payload: {
|
|
3336
|
+
commandBatchId,
|
|
3337
|
+
commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
|
|
3338
|
+
},
|
|
3339
|
+
currentStepOrder: payload.stepIndex
|
|
3340
|
+
});
|
|
3341
|
+
}
|
|
3342
|
+
if (typeof payload.stepIndex === "number") {
|
|
3343
|
+
const prevStep = stepIndexRef.current;
|
|
3344
|
+
stepIndexRef.current = payload.stepIndex;
|
|
3345
|
+
setCurrentStepIndex(payload.stepIndex);
|
|
3346
|
+
if (payload.stepIndex !== prevStep) {
|
|
3347
|
+
const tour = tourRef.current;
|
|
3348
|
+
const total = tour?.steps?.length ?? 0;
|
|
3349
|
+
if (tour && total > 0) {
|
|
3350
|
+
onStepChangeRef.current?.(payload.stepIndex, total, tour);
|
|
3267
3351
|
}
|
|
3268
|
-
}
|
|
3269
|
-
|
|
3270
|
-
const executionToken = ++activeExecutionTokenRef.current;
|
|
3271
|
-
const activeTourId = tourRef.current?.id;
|
|
3272
|
-
const activePreviewRunId = previewRunIdRef.current;
|
|
3273
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3352
|
+
}
|
|
3353
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3274
3354
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3275
3355
|
stepOrder: payload.stepIndex,
|
|
3276
|
-
eventType: "
|
|
3356
|
+
eventType: "step_started",
|
|
3277
3357
|
payload: {
|
|
3278
|
-
|
|
3279
|
-
|
|
3358
|
+
previousStepOrder: prevStep,
|
|
3359
|
+
stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
|
|
3280
3360
|
},
|
|
3281
3361
|
currentStepOrder: payload.stepIndex
|
|
3282
3362
|
});
|
|
3283
3363
|
}
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
}
|
|
3307
|
-
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3308
|
-
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3309
|
-
commandInFlightRef.current = false;
|
|
3310
|
-
emitIfOpen("tour:action_result", {
|
|
3311
|
-
success: false,
|
|
3312
|
-
reason: "invalid_commands",
|
|
3313
|
-
commandBatchId,
|
|
3314
|
-
runId: runIdRef.current,
|
|
3315
|
-
turnId: turnIdRef.current
|
|
3316
|
-
});
|
|
3317
|
-
clearCommandBatchId();
|
|
3318
|
-
return;
|
|
3364
|
+
}
|
|
3365
|
+
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3366
|
+
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3367
|
+
commandInFlightRef.current = false;
|
|
3368
|
+
emitIfOpen("tour:action_result", {
|
|
3369
|
+
success: false,
|
|
3370
|
+
reason: "invalid_commands",
|
|
3371
|
+
commandBatchId,
|
|
3372
|
+
runId: runIdRef.current,
|
|
3373
|
+
turnId: turnIdRef.current
|
|
3374
|
+
});
|
|
3375
|
+
clearCommandBatchId();
|
|
3376
|
+
return;
|
|
3377
|
+
}
|
|
3378
|
+
let shouldWait = false;
|
|
3379
|
+
const results = [];
|
|
3380
|
+
let batchPreferredWaitTarget = null;
|
|
3381
|
+
const assertNotInterrupted = () => {
|
|
3382
|
+
if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
|
|
3383
|
+
const error = new Error("interrupted");
|
|
3384
|
+
error.code = "INTERRUPTED";
|
|
3385
|
+
throw error;
|
|
3319
3386
|
}
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
}
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3335
|
-
onRetry: () => {
|
|
3336
|
-
assertNotInterrupted();
|
|
3337
|
-
},
|
|
3338
|
-
resolve: async () => {
|
|
3339
|
-
let targetEl = null;
|
|
3340
|
-
if (params.uid) {
|
|
3341
|
-
const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3342
|
-
targetEl = getElementByUid2(params.uid);
|
|
3343
|
-
}
|
|
3344
|
-
if (!targetEl) {
|
|
3345
|
-
targetEl = resolveElementFromHints({
|
|
3346
|
-
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3347
|
-
testId: params.testId ?? fallbackHints?.testId,
|
|
3348
|
-
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3349
|
-
}, fallbackStep, tagStore);
|
|
3350
|
-
}
|
|
3351
|
-
return targetEl;
|
|
3352
|
-
}
|
|
3353
|
-
});
|
|
3354
|
-
};
|
|
3355
|
-
const executeOne = async (action) => {
|
|
3356
|
-
assertNotInterrupted();
|
|
3357
|
-
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3358
|
-
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3359
|
-
const executeTimeline = async (params = {}) => {
|
|
3360
|
-
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3361
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
3362
|
-
assertNotInterrupted();
|
|
3363
|
-
const segment = segments[index];
|
|
3364
|
-
const segmentText = (segment?.text ?? "").trim();
|
|
3365
|
-
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3366
|
-
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3367
|
-
if (segmentDelayMs > 0) {
|
|
3368
|
-
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3369
|
-
}
|
|
3370
|
-
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3371
|
-
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3372
|
-
if (!gateTarget) {
|
|
3373
|
-
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3374
|
-
}
|
|
3375
|
-
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3376
|
-
}
|
|
3377
|
-
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3378
|
-
showCaption(segmentText);
|
|
3379
|
-
}
|
|
3380
|
-
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3381
|
-
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3382
|
-
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3383
|
-
onNearEnd: nextSegmentText ? () => {
|
|
3384
|
-
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3385
|
-
} : void 0
|
|
3386
|
-
}) : Promise.resolve();
|
|
3387
|
-
const eventsPromise = (async () => {
|
|
3388
|
-
for (const event of events) {
|
|
3389
|
-
assertNotInterrupted();
|
|
3390
|
-
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3391
|
-
if (delayMs > 0) {
|
|
3392
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3393
|
-
}
|
|
3394
|
-
if (event?.action) {
|
|
3395
|
-
await executeOne(event.action);
|
|
3396
|
-
}
|
|
3397
|
-
}
|
|
3398
|
-
})();
|
|
3399
|
-
await Promise.all([speechPromise, eventsPromise]);
|
|
3400
|
-
}
|
|
3401
|
-
if (params.removeHighlightAtEnd !== false) {
|
|
3402
|
-
removeHighlight();
|
|
3403
|
-
}
|
|
3404
|
-
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3405
|
-
removeCaption();
|
|
3406
|
-
}
|
|
3407
|
-
return !!params.waitForInput;
|
|
3408
|
-
};
|
|
3409
|
-
if (action.type === "speak") {
|
|
3410
|
-
const text = action.params?.text ?? "";
|
|
3411
|
-
if (!text.trim()) {
|
|
3412
|
-
return { result: null };
|
|
3387
|
+
};
|
|
3388
|
+
const resolveTargetElement2 = async (params = {}, fallbackStep) => {
|
|
3389
|
+
const fallbackHints = fallbackStep?.element ?? null;
|
|
3390
|
+
return retryLookup({
|
|
3391
|
+
timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
|
|
3392
|
+
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3393
|
+
onRetry: () => {
|
|
3394
|
+
assertNotInterrupted();
|
|
3395
|
+
},
|
|
3396
|
+
resolve: async () => {
|
|
3397
|
+
let targetEl = null;
|
|
3398
|
+
if (params.uid) {
|
|
3399
|
+
const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3400
|
+
targetEl = getElementByUid2(params.uid);
|
|
3413
3401
|
}
|
|
3414
|
-
if (
|
|
3415
|
-
|
|
3402
|
+
if (!targetEl) {
|
|
3403
|
+
targetEl = resolveElementFromHints({
|
|
3404
|
+
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3405
|
+
testId: params.testId ?? fallbackHints?.testId,
|
|
3406
|
+
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3407
|
+
}, fallbackStep, tagStore);
|
|
3416
3408
|
}
|
|
3417
|
-
|
|
3418
|
-
// Schedule narration immediately so the client keeps its speculative,
|
|
3419
|
-
// low-latency behavior, but defer batch completion until playback settles.
|
|
3420
|
-
waitForCompletion: false,
|
|
3421
|
-
interrupt: action.params?.interrupt
|
|
3422
|
-
});
|
|
3423
|
-
return { result: text, settlePromise };
|
|
3409
|
+
return targetEl;
|
|
3424
3410
|
}
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3411
|
+
});
|
|
3412
|
+
};
|
|
3413
|
+
const executeOne = async (action) => {
|
|
3414
|
+
assertNotInterrupted();
|
|
3415
|
+
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3416
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3417
|
+
const executeTimeline = async (params = {}) => {
|
|
3418
|
+
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3419
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
3420
|
+
assertNotInterrupted();
|
|
3421
|
+
const segment = segments[index];
|
|
3422
|
+
const segmentText = (segment?.text ?? "").trim();
|
|
3423
|
+
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3424
|
+
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3425
|
+
if (segmentDelayMs > 0) {
|
|
3426
|
+
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3427
|
+
}
|
|
3428
|
+
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3429
|
+
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3430
|
+
if (!gateTarget) {
|
|
3431
|
+
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3435
3432
|
}
|
|
3436
|
-
|
|
3437
|
-
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3438
|
-
return { result: "highlighted" };
|
|
3433
|
+
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3439
3434
|
}
|
|
3440
|
-
if (
|
|
3441
|
-
|
|
3442
|
-
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3443
|
-
);
|
|
3435
|
+
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3436
|
+
showCaption(segmentText);
|
|
3444
3437
|
}
|
|
3445
|
-
|
|
3438
|
+
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3439
|
+
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3440
|
+
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3441
|
+
onNearEnd: nextSegmentText ? () => {
|
|
3442
|
+
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3443
|
+
} : void 0
|
|
3444
|
+
}) : Promise.resolve();
|
|
3445
|
+
const eventsPromise = (async () => {
|
|
3446
|
+
for (const event of events) {
|
|
3447
|
+
assertNotInterrupted();
|
|
3448
|
+
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3449
|
+
if (delayMs > 0) {
|
|
3450
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3451
|
+
}
|
|
3452
|
+
if (event?.action) {
|
|
3453
|
+
await executeOne(event.action);
|
|
3454
|
+
}
|
|
3455
|
+
}
|
|
3456
|
+
})();
|
|
3457
|
+
await Promise.all([speechPromise, eventsPromise]);
|
|
3446
3458
|
}
|
|
3447
|
-
if (
|
|
3459
|
+
if (params.removeHighlightAtEnd !== false) {
|
|
3448
3460
|
removeHighlight();
|
|
3449
|
-
return { result: "highlight_removed" };
|
|
3450
3461
|
}
|
|
3451
|
-
if (
|
|
3452
|
-
|
|
3453
|
-
if (!targetEl) {
|
|
3454
|
-
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3455
|
-
throw new Error(
|
|
3456
|
-
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3457
|
-
);
|
|
3458
|
-
}
|
|
3459
|
-
removeHighlight();
|
|
3460
|
-
await performInteractiveClick(targetEl);
|
|
3461
|
-
const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3462
|
-
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3463
|
-
return { result: "clicked" };
|
|
3462
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3463
|
+
removeCaption();
|
|
3464
3464
|
}
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
}
|
|
3472
|
-
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3473
|
-
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3474
|
-
await performInteractiveFill(targetEl, value);
|
|
3475
|
-
return { result: value };
|
|
3465
|
+
return !!params.waitForInput;
|
|
3466
|
+
};
|
|
3467
|
+
if (action.type === "speak") {
|
|
3468
|
+
const text = action.params?.text ?? "";
|
|
3469
|
+
if (!text.trim()) {
|
|
3470
|
+
return { result: null };
|
|
3476
3471
|
}
|
|
3477
|
-
if (
|
|
3478
|
-
|
|
3479
|
-
const html2canvas2 = html2canvasModule.default;
|
|
3480
|
-
const canvas = await html2canvas2(document.body, {
|
|
3481
|
-
useCORS: true,
|
|
3482
|
-
allowTaint: true,
|
|
3483
|
-
scale: Math.min(window.devicePixelRatio, 2),
|
|
3484
|
-
width: window.innerWidth,
|
|
3485
|
-
height: window.innerHeight,
|
|
3486
|
-
x: window.scrollX,
|
|
3487
|
-
y: window.scrollY,
|
|
3488
|
-
logging: false
|
|
3489
|
-
});
|
|
3490
|
-
return { result: canvas.toDataURL("image/png") };
|
|
3472
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3473
|
+
showCaption(text);
|
|
3491
3474
|
}
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3475
|
+
const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
|
|
3476
|
+
// Schedule narration immediately so the client keeps its speculative,
|
|
3477
|
+
// low-latency behavior, but defer batch completion until playback settles.
|
|
3478
|
+
waitForCompletion: false,
|
|
3479
|
+
interrupt: action.params?.interrupt
|
|
3480
|
+
});
|
|
3481
|
+
return { result: text, settlePromise };
|
|
3482
|
+
}
|
|
3483
|
+
if (action.type === "play_timeline") {
|
|
3484
|
+
const timelineShouldWait = await executeTimeline(action.params);
|
|
3485
|
+
if (timelineShouldWait) shouldWait = true;
|
|
3486
|
+
return { result: "timeline_executed" };
|
|
3487
|
+
}
|
|
3488
|
+
if (action.type === "highlight_element") {
|
|
3489
|
+
const resolvedEl = await resolveTargetElement2(action.params, currentStep);
|
|
3490
|
+
if (resolvedEl) {
|
|
3491
|
+
if (isEditableWaitTarget(resolvedEl)) {
|
|
3492
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
|
|
3496
3493
|
}
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
return { result: nextUrl };
|
|
3494
|
+
showHighlight(resolvedEl, action.params?.label || action.label);
|
|
3495
|
+
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3496
|
+
return { result: "highlighted" };
|
|
3501
3497
|
}
|
|
3502
|
-
if (action.
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
}
|
|
3507
|
-
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3508
|
-
const res = await fetch(url, {
|
|
3509
|
-
method: "POST",
|
|
3510
|
-
headers: { "Content-Type": "application/json" },
|
|
3511
|
-
body: JSON.stringify({
|
|
3512
|
-
command: action.params?.command,
|
|
3513
|
-
socketId: agentSocketId
|
|
3514
|
-
})
|
|
3515
|
-
});
|
|
3516
|
-
if (!res.ok) {
|
|
3517
|
-
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3518
|
-
}
|
|
3519
|
-
if (action.params?.wait !== false) {
|
|
3520
|
-
await res.json();
|
|
3521
|
-
}
|
|
3522
|
-
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3523
|
-
await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3524
|
-
await syncAOM();
|
|
3525
|
-
return { result: action.params?.command ?? "executed" };
|
|
3498
|
+
if (!action.params?.optional) {
|
|
3499
|
+
throw new Error(
|
|
3500
|
+
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3501
|
+
);
|
|
3526
3502
|
}
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3503
|
+
return { result: "highlight_optional_miss" };
|
|
3504
|
+
}
|
|
3505
|
+
if (action.type === "remove_highlight") {
|
|
3506
|
+
removeHighlight();
|
|
3507
|
+
return { result: "highlight_removed" };
|
|
3508
|
+
}
|
|
3509
|
+
if (action.type === "click_element") {
|
|
3510
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3511
|
+
if (!targetEl) {
|
|
3512
|
+
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3513
|
+
throw new Error(
|
|
3514
|
+
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3515
|
+
);
|
|
3535
3516
|
}
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3517
|
+
removeHighlight();
|
|
3518
|
+
await performInteractiveClick(targetEl);
|
|
3519
|
+
const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3520
|
+
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3521
|
+
return { result: "clicked" };
|
|
3522
|
+
}
|
|
3523
|
+
if (action.type === "fill_input") {
|
|
3524
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3525
|
+
if (!targetEl) {
|
|
3526
|
+
throw new Error(
|
|
3527
|
+
`fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3528
|
+
);
|
|
3539
3529
|
}
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3530
|
+
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3531
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3532
|
+
await performInteractiveFill(targetEl, value);
|
|
3533
|
+
return { result: value };
|
|
3534
|
+
}
|
|
3535
|
+
if (action.type === "take_screenshot") {
|
|
3536
|
+
const html2canvasModule = await import("html2canvas");
|
|
3537
|
+
const html2canvas2 = html2canvasModule.default;
|
|
3538
|
+
const canvas = await html2canvas2(document.body, {
|
|
3539
|
+
useCORS: true,
|
|
3540
|
+
allowTaint: true,
|
|
3541
|
+
scale: Math.min(window.devicePixelRatio, 2),
|
|
3542
|
+
width: window.innerWidth,
|
|
3543
|
+
height: window.innerHeight,
|
|
3544
|
+
x: window.scrollX,
|
|
3545
|
+
y: window.scrollY,
|
|
3546
|
+
logging: false
|
|
3547
|
+
});
|
|
3548
|
+
return { result: canvas.toDataURL("image/png") };
|
|
3549
|
+
}
|
|
3550
|
+
if (action.type === "navigate_to_url") {
|
|
3551
|
+
const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
|
|
3552
|
+
if (!nextUrl) {
|
|
3553
|
+
throw new Error("navigate_to_url missing url");
|
|
3543
3554
|
}
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
const isTerminal = isTerminalAction(command);
|
|
3554
|
-
if (isTerminal) {
|
|
3555
|
-
await Promise.all(pendingUIActions);
|
|
3556
|
-
pendingUIActions.length = 0;
|
|
3557
|
-
}
|
|
3558
|
-
const executionPromise = (async () => {
|
|
3559
|
-
const execution = await executeOne(command);
|
|
3560
|
-
await execution.settlePromise;
|
|
3561
|
-
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3562
|
-
})();
|
|
3563
|
-
if (isTerminal) {
|
|
3564
|
-
await executionPromise;
|
|
3565
|
-
} else {
|
|
3566
|
-
pendingUIActions.push(executionPromise);
|
|
3567
|
-
}
|
|
3555
|
+
await navigateToTourUrl(nextUrl);
|
|
3556
|
+
const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3557
|
+
await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3558
|
+
return { result: nextUrl };
|
|
3559
|
+
}
|
|
3560
|
+
if (action.type === "execute_agent_action") {
|
|
3561
|
+
const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
|
|
3562
|
+
if (!agentSocketId) {
|
|
3563
|
+
throw new Error("No socketId available for execute_agent_action");
|
|
3568
3564
|
}
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3565
|
+
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3566
|
+
const res = await fetch(url, {
|
|
3567
|
+
method: "POST",
|
|
3568
|
+
headers: { "Content-Type": "application/json" },
|
|
3569
|
+
body: JSON.stringify({
|
|
3570
|
+
command: action.params?.command,
|
|
3571
|
+
socketId: agentSocketId
|
|
3572
|
+
})
|
|
3572
3573
|
});
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
commandInFlightRef.current = false;
|
|
3576
|
-
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3577
|
-
if (interrupted) {
|
|
3578
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3579
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3580
|
-
stepOrder: stepIndexRef.current,
|
|
3581
|
-
eventType: "command_batch_interrupted",
|
|
3582
|
-
payload: {
|
|
3583
|
-
commandBatchId,
|
|
3584
|
-
partialResults: results
|
|
3585
|
-
},
|
|
3586
|
-
currentStepOrder: stepIndexRef.current
|
|
3587
|
-
});
|
|
3588
|
-
}
|
|
3589
|
-
emitIfOpen("tour:action_result", {
|
|
3590
|
-
success: true,
|
|
3591
|
-
interrupted: true,
|
|
3592
|
-
results,
|
|
3593
|
-
commandBatchId,
|
|
3594
|
-
runId: runIdRef.current,
|
|
3595
|
-
turnId: turnIdRef.current
|
|
3596
|
-
});
|
|
3597
|
-
clearCommandBatchId();
|
|
3598
|
-
return;
|
|
3574
|
+
if (!res.ok) {
|
|
3575
|
+
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3599
3576
|
}
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3603
|
-
stepOrder: stepIndexRef.current,
|
|
3604
|
-
eventType: "command_batch_failed",
|
|
3605
|
-
payload: {
|
|
3606
|
-
commandBatchId,
|
|
3607
|
-
error: String(err),
|
|
3608
|
-
partialResults: results
|
|
3609
|
-
},
|
|
3610
|
-
status: "active",
|
|
3611
|
-
currentStepOrder: stepIndexRef.current
|
|
3612
|
-
});
|
|
3577
|
+
if (action.params?.wait !== false) {
|
|
3578
|
+
await res.json();
|
|
3613
3579
|
}
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
results,
|
|
3619
|
-
commandBatchId,
|
|
3620
|
-
runId: runIdRef.current,
|
|
3621
|
-
turnId: turnIdRef.current
|
|
3622
|
-
});
|
|
3623
|
-
clearCommandBatchId();
|
|
3624
|
-
return;
|
|
3580
|
+
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3581
|
+
await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3582
|
+
await syncAOM();
|
|
3583
|
+
return { result: action.params?.command ?? "executed" };
|
|
3625
3584
|
}
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3631
|
-
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3632
|
-
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3633
|
-
let manualWaitPromise = null;
|
|
3634
|
-
let manualWaitKind = null;
|
|
3635
|
-
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3636
|
-
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3637
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3638
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3639
|
-
if (waitTargetHints) {
|
|
3640
|
-
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3641
|
-
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3642
|
-
manualWaitTarget = preferredWaitTarget;
|
|
3643
|
-
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3644
|
-
}
|
|
3645
|
-
if (manualWaitTarget) {
|
|
3646
|
-
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3647
|
-
manualWaitPromise = manualWait.promise;
|
|
3648
|
-
manualWaitKind = manualWait.kind;
|
|
3649
|
-
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3650
|
-
}
|
|
3585
|
+
if (action.type === "wait_for_user_action") {
|
|
3586
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3587
|
+
if (!targetEl) {
|
|
3588
|
+
throw new Error("wait_for_user_action target not found");
|
|
3651
3589
|
}
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3590
|
+
const eventName = action.params?.event ?? "click";
|
|
3591
|
+
await waitForUserAction(targetEl, eventName);
|
|
3592
|
+
return { result: `waited_for_${eventName}` };
|
|
3593
|
+
}
|
|
3594
|
+
if (action.type === "wait_for_input") {
|
|
3595
|
+
shouldWait = true;
|
|
3596
|
+
return { result: "waiting_for_input" };
|
|
3597
|
+
}
|
|
3598
|
+
if (action.type === "end_tour") {
|
|
3599
|
+
handleTourEnd();
|
|
3600
|
+
return { result: "ended" };
|
|
3601
|
+
}
|
|
3602
|
+
console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
|
|
3603
|
+
return { result: "unknown_action_skipped" };
|
|
3604
|
+
};
|
|
3605
|
+
try {
|
|
3606
|
+
const resultsBuffer = new Array(payload.commands.length);
|
|
3607
|
+
const pendingUIActions = [];
|
|
3608
|
+
for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
|
|
3609
|
+
const command = payload.commands[commandIndex];
|
|
3610
|
+
assertNotInterrupted();
|
|
3611
|
+
const isTerminal = isTerminalAction(command);
|
|
3612
|
+
if (isTerminal) {
|
|
3613
|
+
await Promise.all(pendingUIActions);
|
|
3614
|
+
pendingUIActions.length = 0;
|
|
3658
3615
|
}
|
|
3659
|
-
|
|
3660
|
-
const
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3669
|
-
}
|
|
3616
|
+
const executionPromise = (async () => {
|
|
3617
|
+
const execution = await executeOne(command);
|
|
3618
|
+
await execution.settlePromise;
|
|
3619
|
+
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3620
|
+
})();
|
|
3621
|
+
if (isTerminal) {
|
|
3622
|
+
await executionPromise;
|
|
3623
|
+
} else {
|
|
3624
|
+
pendingUIActions.push(executionPromise);
|
|
3670
3625
|
}
|
|
3671
|
-
|
|
3626
|
+
}
|
|
3627
|
+
await Promise.all(pendingUIActions);
|
|
3628
|
+
resultsBuffer.forEach((res) => {
|
|
3629
|
+
if (res) results.push(res);
|
|
3630
|
+
});
|
|
3631
|
+
await syncAOM();
|
|
3632
|
+
} catch (err) {
|
|
3633
|
+
commandInFlightRef.current = false;
|
|
3634
|
+
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3635
|
+
if (interrupted) {
|
|
3672
3636
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3673
3637
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3674
3638
|
stepOrder: stepIndexRef.current,
|
|
3675
|
-
eventType: "
|
|
3639
|
+
eventType: "command_batch_interrupted",
|
|
3676
3640
|
payload: {
|
|
3677
3641
|
commandBatchId,
|
|
3678
|
-
results
|
|
3642
|
+
partialResults: results
|
|
3679
3643
|
},
|
|
3680
3644
|
currentStepOrder: stepIndexRef.current
|
|
3681
3645
|
});
|
|
3682
3646
|
}
|
|
3683
3647
|
emitIfOpen("tour:action_result", {
|
|
3684
3648
|
success: true,
|
|
3685
|
-
|
|
3649
|
+
interrupted: true,
|
|
3686
3650
|
results,
|
|
3687
3651
|
commandBatchId,
|
|
3688
3652
|
runId: runIdRef.current,
|
|
3689
3653
|
turnId: turnIdRef.current
|
|
3690
3654
|
});
|
|
3691
3655
|
clearCommandBatchId();
|
|
3692
|
-
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3693
|
-
if (pendingInputBufRef.current) {
|
|
3694
|
-
const flushed = pendingInputBufRef.current;
|
|
3695
|
-
pendingInputBufRef.current = null;
|
|
3696
|
-
resolve(flushed);
|
|
3697
|
-
return;
|
|
3698
|
-
}
|
|
3699
|
-
voiceInputResolveRef.current = (text) => {
|
|
3700
|
-
voiceInputResolveRef.current = null;
|
|
3701
|
-
resolve(text);
|
|
3702
|
-
};
|
|
3703
|
-
});
|
|
3704
|
-
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3705
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3706
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3707
|
-
voiceInputResolveRef.current = null;
|
|
3708
|
-
setPlaybackState("executing");
|
|
3709
|
-
const transcript = userText.trim();
|
|
3710
|
-
if (!transcript) {
|
|
3711
|
-
return;
|
|
3712
|
-
}
|
|
3713
|
-
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3714
|
-
await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
|
|
3715
|
-
await syncAOM();
|
|
3716
|
-
emitIfOpen("tour:user_input", {
|
|
3717
|
-
transcript,
|
|
3718
|
-
runId: runIdRef.current,
|
|
3719
|
-
turnId: turnIdRef.current
|
|
3720
|
-
});
|
|
3721
|
-
});
|
|
3722
3656
|
return;
|
|
3723
3657
|
}
|
|
3658
|
+
console.error("[TourClient] Command batch execution failed:", err);
|
|
3724
3659
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3725
3660
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3726
3661
|
stepOrder: stepIndexRef.current,
|
|
3727
|
-
eventType: "
|
|
3662
|
+
eventType: "command_batch_failed",
|
|
3728
3663
|
payload: {
|
|
3729
3664
|
commandBatchId,
|
|
3730
|
-
|
|
3665
|
+
error: String(err),
|
|
3666
|
+
partialResults: results
|
|
3731
3667
|
},
|
|
3668
|
+
status: "active",
|
|
3732
3669
|
currentStepOrder: stepIndexRef.current
|
|
3733
3670
|
});
|
|
3734
3671
|
}
|
|
3735
3672
|
emitIfOpen("tour:action_result", {
|
|
3736
|
-
success:
|
|
3673
|
+
success: false,
|
|
3674
|
+
reason: "execution_error",
|
|
3675
|
+
error: String(err),
|
|
3737
3676
|
results,
|
|
3738
3677
|
commandBatchId,
|
|
3739
3678
|
runId: runIdRef.current,
|
|
3740
3679
|
turnId: turnIdRef.current
|
|
3741
3680
|
});
|
|
3742
3681
|
clearCommandBatchId();
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
const
|
|
3748
|
-
const
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3682
|
+
return;
|
|
3683
|
+
}
|
|
3684
|
+
commandInFlightRef.current = false;
|
|
3685
|
+
if (shouldWait && !skipRequestedRef.current) {
|
|
3686
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3687
|
+
const waitCondition = currentStep?.onboarding?.waitCondition;
|
|
3688
|
+
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3689
|
+
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3690
|
+
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3691
|
+
let manualWaitPromise = null;
|
|
3692
|
+
let manualWaitKind = null;
|
|
3693
|
+
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3694
|
+
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3695
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3696
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3697
|
+
if (waitTargetHints) {
|
|
3698
|
+
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3699
|
+
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3700
|
+
manualWaitTarget = preferredWaitTarget;
|
|
3701
|
+
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3702
|
+
}
|
|
3703
|
+
if (manualWaitTarget) {
|
|
3704
|
+
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3705
|
+
manualWaitPromise = manualWait.promise;
|
|
3706
|
+
manualWaitKind = manualWait.kind;
|
|
3707
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3708
|
+
}
|
|
3709
|
+
}
|
|
3710
|
+
if (!manualWaitPromise && preferredWaitTarget) {
|
|
3711
|
+
const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
|
|
3712
|
+
manualWaitPromise = manualWait.promise;
|
|
3713
|
+
manualWaitKind = manualWait.kind;
|
|
3714
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3715
|
+
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3752
3716
|
}
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
|
|
3765
|
-
|
|
3766
|
-
|
|
3717
|
+
if (!manualWaitPromise && inputLikeWait) {
|
|
3718
|
+
const firstInput = document.querySelector(
|
|
3719
|
+
'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
|
|
3720
|
+
);
|
|
3721
|
+
if (firstInput) {
|
|
3722
|
+
const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
|
|
3723
|
+
manualWaitPromise = manualWait.promise;
|
|
3724
|
+
manualWaitKind = manualWait.kind;
|
|
3725
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3726
|
+
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3727
|
+
}
|
|
3728
|
+
}
|
|
3729
|
+
setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
|
|
3730
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3731
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3732
|
+
stepOrder: stepIndexRef.current,
|
|
3733
|
+
eventType: "waiting_for_input",
|
|
3767
3734
|
payload: {
|
|
3768
|
-
|
|
3769
|
-
|
|
3735
|
+
commandBatchId,
|
|
3736
|
+
results
|
|
3770
3737
|
},
|
|
3771
|
-
currentStepOrder:
|
|
3738
|
+
currentStepOrder: stepIndexRef.current
|
|
3772
3739
|
});
|
|
3773
3740
|
}
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3741
|
+
emitIfOpen("tour:action_result", {
|
|
3742
|
+
success: true,
|
|
3743
|
+
waitingForInput: true,
|
|
3744
|
+
results,
|
|
3745
|
+
commandBatchId,
|
|
3746
|
+
runId: runIdRef.current,
|
|
3747
|
+
turnId: turnIdRef.current
|
|
3748
|
+
});
|
|
3749
|
+
clearCommandBatchId();
|
|
3750
|
+
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3751
|
+
if (pendingInputBufRef.current) {
|
|
3752
|
+
const flushed = pendingInputBufRef.current;
|
|
3753
|
+
pendingInputBufRef.current = null;
|
|
3754
|
+
resolve(flushed);
|
|
3755
|
+
return;
|
|
3783
3756
|
}
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3757
|
+
voiceInputResolveRef.current = (text) => {
|
|
3758
|
+
voiceInputResolveRef.current = null;
|
|
3759
|
+
resolve(text);
|
|
3760
|
+
};
|
|
3761
|
+
});
|
|
3762
|
+
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3763
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3764
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3765
|
+
voiceInputResolveRef.current = null;
|
|
3766
|
+
setPlaybackState("executing");
|
|
3767
|
+
const transcript = userText.trim();
|
|
3768
|
+
if (!transcript) {
|
|
3769
|
+
return;
|
|
3770
|
+
}
|
|
3771
|
+
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3772
|
+
await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
|
|
3773
|
+
await syncAOM();
|
|
3774
|
+
emitIfOpen("tour:user_input", {
|
|
3775
|
+
transcript,
|
|
3776
|
+
runId: runIdRef.current,
|
|
3777
|
+
turnId: turnIdRef.current
|
|
3778
|
+
});
|
|
3779
|
+
});
|
|
3780
|
+
return;
|
|
3781
|
+
}
|
|
3782
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3783
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3784
|
+
stepOrder: stepIndexRef.current,
|
|
3785
|
+
eventType: "command_batch_completed",
|
|
3786
|
+
payload: {
|
|
3787
|
+
commandBatchId,
|
|
3788
|
+
results
|
|
3789
|
+
},
|
|
3790
|
+
currentStepOrder: stepIndexRef.current
|
|
3791
|
+
});
|
|
3792
|
+
}
|
|
3793
|
+
emitIfOpen("tour:action_result", {
|
|
3794
|
+
success: true,
|
|
3795
|
+
results,
|
|
3796
|
+
commandBatchId,
|
|
3797
|
+
runId: runIdRef.current,
|
|
3798
|
+
turnId: turnIdRef.current
|
|
3787
3799
|
});
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3800
|
+
clearCommandBatchId();
|
|
3801
|
+
};
|
|
3802
|
+
const handleTourStart = async (tourData) => {
|
|
3803
|
+
if (isActiveRef.current) return;
|
|
3804
|
+
runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
|
|
3805
|
+
const tour = tourData.tourContext ?? tourRef.current;
|
|
3806
|
+
const expType = experienceTypeRef.current;
|
|
3807
|
+
if (tour?.type && tour.type !== expType) {
|
|
3808
|
+
console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
|
|
3809
|
+
return;
|
|
3810
|
+
}
|
|
3811
|
+
skipRequestedRef.current = false;
|
|
3812
|
+
const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
|
|
3813
|
+
isActiveRef.current = true;
|
|
3814
|
+
setIsActive(true);
|
|
3815
|
+
setActiveTour(tour ?? null);
|
|
3816
|
+
tourRef.current = tour ?? null;
|
|
3817
|
+
setTotalSteps(total);
|
|
3818
|
+
stepIndexRef.current = 0;
|
|
3819
|
+
setCurrentStepIndex(0);
|
|
3820
|
+
setPlaybackState("intro");
|
|
3821
|
+
if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
|
|
3822
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
|
|
3823
|
+
stepOrder: 0,
|
|
3824
|
+
eventType: "tour_started",
|
|
3825
|
+
payload: {
|
|
3826
|
+
totalSteps: total,
|
|
3827
|
+
source: "sdk_test_preview"
|
|
3828
|
+
},
|
|
3829
|
+
currentStepOrder: 0
|
|
3830
|
+
});
|
|
3831
|
+
}
|
|
3832
|
+
try {
|
|
3833
|
+
const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3834
|
+
const aom = generateMinifiedAOM2();
|
|
3835
|
+
if (socketRef.current === socket) {
|
|
3836
|
+
emitSocketEvent(socket, "tour:sync_dom", {
|
|
3837
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3838
|
+
aom: aom.nodes,
|
|
3839
|
+
domSummary: captureDomSummary()
|
|
3840
|
+
});
|
|
3792
3841
|
}
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3842
|
+
} catch (e) {
|
|
3843
|
+
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3844
|
+
}
|
|
3845
|
+
};
|
|
3846
|
+
const handleTourUpdate = (payload) => {
|
|
3847
|
+
const updatedTour = payload?.tourContext;
|
|
3848
|
+
if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
|
|
3849
|
+
return;
|
|
3850
|
+
}
|
|
3851
|
+
tourRef.current = updatedTour;
|
|
3852
|
+
setActiveTour(updatedTour);
|
|
3853
|
+
const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
|
|
3854
|
+
setTotalSteps(nextTotal);
|
|
3855
|
+
if (typeof payload.currentStepIndex === "number") {
|
|
3856
|
+
const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
|
|
3857
|
+
stepIndexRef.current = clampedStepIndex;
|
|
3858
|
+
setCurrentStepIndex(clampedStepIndex);
|
|
3859
|
+
if (nextTotal > 0) {
|
|
3860
|
+
onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
|
|
3804
3861
|
}
|
|
3805
|
-
}
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
}
|
|
3818
|
-
}
|
|
3862
|
+
}
|
|
3863
|
+
};
|
|
3864
|
+
const handleTourEndEvent = () => {
|
|
3865
|
+
setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
|
|
3866
|
+
handleTourEnd();
|
|
3867
|
+
};
|
|
3868
|
+
const handleDebugLog = (entry) => {
|
|
3869
|
+
const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
|
|
3870
|
+
if (isDev) {
|
|
3871
|
+
console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
|
|
3872
|
+
if (typeof window !== "undefined") {
|
|
3873
|
+
window.dispatchEvent(new CustomEvent("modelnex-debug", {
|
|
3874
|
+
detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
|
|
3875
|
+
}));
|
|
3819
3876
|
}
|
|
3820
|
-
});
|
|
3821
|
-
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3822
|
-
});
|
|
3823
|
-
return () => {
|
|
3824
|
-
cancelled = true;
|
|
3825
|
-
const toClose = createdSocket ?? socketRef.current;
|
|
3826
|
-
if (toClose) {
|
|
3827
|
-
toClose.disconnect();
|
|
3828
3877
|
}
|
|
3829
|
-
|
|
3878
|
+
};
|
|
3879
|
+
socket.on("connect", handleConnect);
|
|
3880
|
+
socket.on("tour:server_state", handleServerState);
|
|
3881
|
+
socket.on("tour:command_cancel", handleCommandCancel);
|
|
3882
|
+
socket.on("tour:command", handleCommand);
|
|
3883
|
+
socket.on("tour:start", handleTourStart);
|
|
3884
|
+
socket.on("tour:update", handleTourUpdate);
|
|
3885
|
+
socket.on("tour:end", handleTourEndEvent);
|
|
3886
|
+
socket.on("tour:debug_log", handleDebugLog);
|
|
3887
|
+
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3888
|
+
return () => {
|
|
3889
|
+
socket.off("connect", handleConnect);
|
|
3890
|
+
socket.off("tour:server_state", handleServerState);
|
|
3891
|
+
socket.off("tour:command_cancel", handleCommandCancel);
|
|
3892
|
+
socket.off("tour:command", handleCommand);
|
|
3893
|
+
socket.off("tour:start", handleTourStart);
|
|
3894
|
+
socket.off("tour:update", handleTourUpdate);
|
|
3895
|
+
socket.off("tour:end", handleTourEndEvent);
|
|
3896
|
+
socket.off("tour:debug_log", handleDebugLog);
|
|
3897
|
+
const toClose = socket;
|
|
3830
3898
|
socketRef.current = null;
|
|
3831
3899
|
setServerState(null);
|
|
3832
3900
|
runIdRef.current = null;
|
|
3833
3901
|
turnIdRef.current = null;
|
|
3902
|
+
tourSocketPool.release(serverUrl, toClose);
|
|
3834
3903
|
};
|
|
3835
|
-
}, [serverUrl,
|
|
3904
|
+
}, [serverUrl, disabled]);
|
|
3836
3905
|
(0, import_react12.useEffect)(() => {
|
|
3837
3906
|
if (disabled) return;
|
|
3838
3907
|
const s = socketRef.current;
|
|
3839
3908
|
const profile = userProfile;
|
|
3840
|
-
if (!
|
|
3841
|
-
|
|
3909
|
+
if (!websiteId || !profile?.userId) return;
|
|
3910
|
+
const timer = setTimeout(() => {
|
|
3911
|
+
emitSocketEvent(s, "tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3912
|
+
}, 150);
|
|
3913
|
+
return () => clearTimeout(timer);
|
|
3842
3914
|
}, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
|
|
3843
3915
|
(0, import_react12.useEffect)(() => {
|
|
3844
3916
|
if (!showCaptions || !isReviewMode) {
|
|
@@ -3846,8 +3918,8 @@ function useTourPlayback({
|
|
|
3846
3918
|
}
|
|
3847
3919
|
}, [showCaptions, isReviewMode]);
|
|
3848
3920
|
(0, import_react12.useEffect)(() => {
|
|
3849
|
-
if (!
|
|
3850
|
-
socketRef.current
|
|
3921
|
+
if (!isActiveRef.current) return;
|
|
3922
|
+
emitSocketEvent(socketRef.current, "tour:client_state", {
|
|
3851
3923
|
runId: runIdRef.current,
|
|
3852
3924
|
turnId: turnIdRef.current,
|
|
3853
3925
|
commandBatchId: activeCommandBatchIdRef.current,
|
|
@@ -3860,17 +3932,17 @@ function useTourPlayback({
|
|
|
3860
3932
|
});
|
|
3861
3933
|
}, [isActive, playbackState, voice.isListening, voice.isSpeaking]);
|
|
3862
3934
|
const syncAOM = (0, import_react12.useCallback)(async () => {
|
|
3863
|
-
if (!
|
|
3935
|
+
if (!isActiveRef.current) return;
|
|
3864
3936
|
const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3865
3937
|
const aom = generateMinifiedAOM2();
|
|
3866
|
-
socketRef.current
|
|
3938
|
+
emitSocketEvent(socketRef.current, "tour:sync_dom", {
|
|
3867
3939
|
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3868
3940
|
aom: aom.nodes,
|
|
3869
3941
|
domSummary: captureDomSummary()
|
|
3870
3942
|
});
|
|
3871
3943
|
}, []);
|
|
3872
3944
|
const interruptExecution = (0, import_react12.useCallback)((transcript) => {
|
|
3873
|
-
if (!socketRef.current
|
|
3945
|
+
if (!isSocketWritable(socketRef.current) || !isActiveRef.current) return false;
|
|
3874
3946
|
if (!commandInFlightRef.current && !voice.isSpeaking) return false;
|
|
3875
3947
|
interruptedForQuestionRef.current = true;
|
|
3876
3948
|
activeExecutionTokenRef.current += 1;
|
|
@@ -3878,21 +3950,22 @@ function useTourPlayback({
|
|
|
3878
3950
|
removeHighlight();
|
|
3879
3951
|
removeCaption();
|
|
3880
3952
|
voice.stopSpeaking();
|
|
3881
|
-
socketRef.current
|
|
3953
|
+
if (emitSocketEvent(socketRef.current, "tour:action_result", {
|
|
3882
3954
|
success: true,
|
|
3883
3955
|
interrupted: true,
|
|
3884
3956
|
transcript,
|
|
3885
3957
|
commandBatchId: activeCommandBatchIdRef.current,
|
|
3886
3958
|
runId: runIdRef.current,
|
|
3887
3959
|
turnId: turnIdRef.current
|
|
3888
|
-
})
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3960
|
+
})) {
|
|
3961
|
+
activeCommandBatchIdRef.current = null;
|
|
3962
|
+
emitSocketEvent(socketRef.current, "tour:user_input", {
|
|
3963
|
+
transcript,
|
|
3964
|
+
interrupted: true,
|
|
3965
|
+
runId: runIdRef.current,
|
|
3966
|
+
turnId: turnIdRef.current
|
|
3967
|
+
});
|
|
3968
|
+
}
|
|
3896
3969
|
setPlaybackState("thinking");
|
|
3897
3970
|
return true;
|
|
3898
3971
|
}, [voice]);
|
|
@@ -3908,9 +3981,7 @@ function useTourPlayback({
|
|
|
3908
3981
|
removeCaption();
|
|
3909
3982
|
voice.stopSpeaking();
|
|
3910
3983
|
voice.stopListening();
|
|
3911
|
-
|
|
3912
|
-
socketRef.current.emit("tour:abort");
|
|
3913
|
-
}
|
|
3984
|
+
emitSocketEvent(socketRef.current, "tour:abort");
|
|
3914
3985
|
if (reviewModeRef.current && tourRef.current?.id && previewRunIdRef.current) {
|
|
3915
3986
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tourRef.current.id, previewRunIdRef.current, websiteId, {
|
|
3916
3987
|
stepOrder: stepIndexRef.current,
|
|
@@ -3982,7 +4053,7 @@ function useTourPlayback({
|
|
|
3982
4053
|
await new Promise((r) => setTimeout(r, 200));
|
|
3983
4054
|
retries++;
|
|
3984
4055
|
}
|
|
3985
|
-
if (!socketRef.current
|
|
4056
|
+
if (!isSocketWritable(socketRef.current)) {
|
|
3986
4057
|
console.warn("[TourClient] Cannot run tour, socket not connected.");
|
|
3987
4058
|
return;
|
|
3988
4059
|
}
|
|
@@ -4010,7 +4081,7 @@ function useTourPlayback({
|
|
|
4010
4081
|
previewRunIdRef.current = null;
|
|
4011
4082
|
}
|
|
4012
4083
|
tourRef.current = tour;
|
|
4013
|
-
socketRef.current
|
|
4084
|
+
emitSocketEvent(socketRef.current, "tour:request_start", {
|
|
4014
4085
|
tourId: tour.id,
|
|
4015
4086
|
previewRunId: previewRunIdRef.current,
|
|
4016
4087
|
tourContext: tour
|
|
@@ -4158,8 +4229,8 @@ function useTourPlayback({
|
|
|
4158
4229
|
const revisionVersion = Number(response?.revision?.versionNumber);
|
|
4159
4230
|
const appliedMessage = Number.isFinite(revisionVersion) && revisionVersion > 0 ? `Applied to the draft as version ${revisionVersion}.` : "Correction applied to the draft.";
|
|
4160
4231
|
setReviewStatusMessage(apply ? appliedMessage : "Correction saved for review.");
|
|
4161
|
-
if (apply && playbackState === "paused" &&
|
|
4162
|
-
socketRef.current
|
|
4232
|
+
if (apply && playbackState === "paused" && isActiveRef.current) {
|
|
4233
|
+
emitSocketEvent(socketRef.current, "tour:resume");
|
|
4163
4234
|
setPlaybackState("executing");
|
|
4164
4235
|
}
|
|
4165
4236
|
} catch (err) {
|
|
@@ -4183,14 +4254,14 @@ function useTourPlayback({
|
|
|
4183
4254
|
stopTour();
|
|
4184
4255
|
}, [stopTour]);
|
|
4185
4256
|
const pauseTour = (0, import_react12.useCallback)(() => {
|
|
4186
|
-
if (socketRef.current
|
|
4187
|
-
socketRef.current
|
|
4257
|
+
if (isSocketWritable(socketRef.current) && isActiveRef.current) {
|
|
4258
|
+
emitSocketEvent(socketRef.current, "tour:pause");
|
|
4188
4259
|
setPlaybackState("paused");
|
|
4189
4260
|
}
|
|
4190
4261
|
}, []);
|
|
4191
4262
|
const resumeTour = (0, import_react12.useCallback)(() => {
|
|
4192
|
-
if (socketRef.current
|
|
4193
|
-
socketRef.current
|
|
4263
|
+
if (isSocketWritable(socketRef.current) && isActiveRef.current) {
|
|
4264
|
+
emitSocketEvent(socketRef.current, "tour:resume");
|
|
4194
4265
|
setPlaybackState("executing");
|
|
4195
4266
|
}
|
|
4196
4267
|
}, []);
|
|
@@ -4212,9 +4283,9 @@ function useTourPlayback({
|
|
|
4212
4283
|
if (voiceInputResolveRef.current) {
|
|
4213
4284
|
console.log("[TourAgent] Resolving loop waiting_voice with text:", text);
|
|
4214
4285
|
voiceInputResolveRef.current(text);
|
|
4215
|
-
} else if (socketRef.current
|
|
4286
|
+
} else if (isSocketWritable(socketRef.current)) {
|
|
4216
4287
|
console.log("[TourAgent] Forwarding ambient voice to server:", text);
|
|
4217
|
-
socketRef.current
|
|
4288
|
+
emitSocketEvent(socketRef.current, "tour:user_input", {
|
|
4218
4289
|
transcript: text,
|
|
4219
4290
|
runId: runIdRef.current,
|
|
4220
4291
|
turnId: turnIdRef.current
|
|
@@ -5462,7 +5533,7 @@ function useVoice(serverUrl) {
|
|
|
5462
5533
|
(async () => {
|
|
5463
5534
|
try {
|
|
5464
5535
|
const ioModule = await import("socket.io-client");
|
|
5465
|
-
const
|
|
5536
|
+
const io3 = ioModule.default || ioModule;
|
|
5466
5537
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
5467
5538
|
audio: {
|
|
5468
5539
|
echoCancellation: true,
|
|
@@ -5484,7 +5555,7 @@ function useVoice(serverUrl) {
|
|
|
5484
5555
|
audioBitsPerSecond: 128e3
|
|
5485
5556
|
});
|
|
5486
5557
|
mediaRecorderRef.current = recorder;
|
|
5487
|
-
const socket =
|
|
5558
|
+
const socket = io3(serverUrl, {
|
|
5488
5559
|
path: "/socket.io",
|
|
5489
5560
|
transports: resolveSocketIoTransports(serverUrl, "websocket-first")
|
|
5490
5561
|
});
|
|
@@ -11005,7 +11076,7 @@ var ModelNexProvider = ({
|
|
|
11005
11076
|
socketId,
|
|
11006
11077
|
devMode
|
|
11007
11078
|
}),
|
|
11008
|
-
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
|
|
11079
|
+
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
|
|
11009
11080
|
);
|
|
11010
11081
|
return import_react21.default.createElement(
|
|
11011
11082
|
ModelNexContext.Provider,
|