@modelnex/sdk 0.5.15 → 0.5.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +647 -576
- package/dist/index.mjs +647 -576
- package/package.json +11 -12
- package/dist/aom-GA6W42DG.mjs +0 -71
- package/dist/aom-J6NYMGDW.mjs +0 -69
- package/dist/chunk-6DZX6EAA.mjs +0 -37
- package/dist/dom-sync-Y7Z7TOU6.mjs +0 -57
package/dist/index.mjs
CHANGED
|
@@ -2438,6 +2438,76 @@ async function retryLookup({
|
|
|
2438
2438
|
}
|
|
2439
2439
|
}
|
|
2440
2440
|
|
|
2441
|
+
// src/utils/tour-socket-pool.ts
|
|
2442
|
+
import { io as io2 } from "socket.io-client";
|
|
2443
|
+
function isSocketWritable(socket) {
|
|
2444
|
+
if (!socket?.connected) return false;
|
|
2445
|
+
const engine = socket.io?.engine;
|
|
2446
|
+
if (!engine) return true;
|
|
2447
|
+
if (typeof engine.readyState === "string" && engine.readyState !== "open") return false;
|
|
2448
|
+
if (engine.transport && "writable" in engine.transport && engine.transport.writable === false) return false;
|
|
2449
|
+
return true;
|
|
2450
|
+
}
|
|
2451
|
+
function emitSocketEvent(socket, event, payload) {
|
|
2452
|
+
if (!isSocketWritable(socket)) return false;
|
|
2453
|
+
socket.emit(event, payload);
|
|
2454
|
+
return true;
|
|
2455
|
+
}
|
|
2456
|
+
function createTourSocketPool({
|
|
2457
|
+
createSocket = (serverUrl) => io2(serverUrl, {
|
|
2458
|
+
path: "/socket.io",
|
|
2459
|
+
transports: resolveSocketIoTransports(serverUrl, "polling-first"),
|
|
2460
|
+
autoConnect: true,
|
|
2461
|
+
reconnection: true,
|
|
2462
|
+
reconnectionAttempts: 10,
|
|
2463
|
+
reconnectionDelay: 1e3
|
|
2464
|
+
}),
|
|
2465
|
+
releaseDelayMs = 2500,
|
|
2466
|
+
scheduleRelease = (callback, delayMs) => setTimeout(callback, delayMs),
|
|
2467
|
+
cancelRelease = (timer) => clearTimeout(timer)
|
|
2468
|
+
} = {}) {
|
|
2469
|
+
const pooledSockets = /* @__PURE__ */ new Map();
|
|
2470
|
+
return {
|
|
2471
|
+
acquire(serverUrl) {
|
|
2472
|
+
const pooled = pooledSockets.get(serverUrl);
|
|
2473
|
+
if (pooled) {
|
|
2474
|
+
if (pooled.releaseTimer) {
|
|
2475
|
+
cancelRelease(pooled.releaseTimer);
|
|
2476
|
+
pooled.releaseTimer = null;
|
|
2477
|
+
}
|
|
2478
|
+
pooled.leases += 1;
|
|
2479
|
+
return pooled.socket;
|
|
2480
|
+
}
|
|
2481
|
+
const socket = createSocket(serverUrl);
|
|
2482
|
+
pooledSockets.set(serverUrl, {
|
|
2483
|
+
socket,
|
|
2484
|
+
leases: 1,
|
|
2485
|
+
releaseTimer: null
|
|
2486
|
+
});
|
|
2487
|
+
return socket;
|
|
2488
|
+
},
|
|
2489
|
+
release(serverUrl, socket) {
|
|
2490
|
+
const pooled = pooledSockets.get(serverUrl);
|
|
2491
|
+
if (!pooled || pooled.socket !== socket) return;
|
|
2492
|
+
pooled.leases = Math.max(0, pooled.leases - 1);
|
|
2493
|
+
if (pooled.leases > 0) return;
|
|
2494
|
+
pooled.releaseTimer = scheduleRelease(() => {
|
|
2495
|
+
const latest = pooledSockets.get(serverUrl);
|
|
2496
|
+
if (!latest || latest !== pooled || latest.leases > 0) return;
|
|
2497
|
+
latest.socket.disconnect();
|
|
2498
|
+
pooledSockets.delete(serverUrl);
|
|
2499
|
+
}, releaseDelayMs);
|
|
2500
|
+
},
|
|
2501
|
+
// Test-only introspection
|
|
2502
|
+
getSnapshot(serverUrl) {
|
|
2503
|
+
const pooled = pooledSockets.get(serverUrl);
|
|
2504
|
+
if (!pooled) return null;
|
|
2505
|
+
return { leases: pooled.leases, socket: pooled.socket };
|
|
2506
|
+
}
|
|
2507
|
+
};
|
|
2508
|
+
}
|
|
2509
|
+
var tourSocketPool = createTourSocketPool();
|
|
2510
|
+
|
|
2441
2511
|
// src/hooks/useTourPlayback.ts
|
|
2442
2512
|
function resolveElement(step) {
|
|
2443
2513
|
const el = step.element;
|
|
@@ -2974,11 +3044,13 @@ function useTourPlayback({
|
|
|
2974
3044
|
const socketRef = useRef8(null);
|
|
2975
3045
|
const socketIdRef = useRef8(socketId);
|
|
2976
3046
|
const commandUrlRef = useRef8(commandUrl);
|
|
3047
|
+
const websiteIdRef = useRef8(websiteId);
|
|
2977
3048
|
const onStepChangeRef = useRef8(onStepChange);
|
|
2978
3049
|
const isActiveRef = useRef8(false);
|
|
2979
3050
|
const activeCommandBatchIdRef = useRef8(null);
|
|
2980
3051
|
socketIdRef.current = socketId;
|
|
2981
3052
|
commandUrlRef.current = commandUrl;
|
|
3053
|
+
websiteIdRef.current = websiteId;
|
|
2982
3054
|
onStepChangeRef.current = onStepChange;
|
|
2983
3055
|
isActiveRef.current = isActive;
|
|
2984
3056
|
reviewModeRef.current = isReviewMode;
|
|
@@ -2989,646 +3061,646 @@ function useTourPlayback({
|
|
|
2989
3061
|
useEffect11(() => {
|
|
2990
3062
|
if (disabled) return;
|
|
2991
3063
|
if (typeof window === "undefined") return;
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
const
|
|
2997
|
-
const
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
transports: resolveSocketIoTransports(serverUrl, "polling-first")
|
|
3001
|
-
});
|
|
3002
|
-
if (cancelled) {
|
|
3003
|
-
socket.disconnect();
|
|
3004
|
-
return;
|
|
3064
|
+
const socket = tourSocketPool.acquire(serverUrl);
|
|
3065
|
+
socketRef.current = socket;
|
|
3066
|
+
const handleConnect = () => {
|
|
3067
|
+
console.log("[TourClient] Connected to tour agent server:", socket.id);
|
|
3068
|
+
const profile = userProfileRef.current;
|
|
3069
|
+
const currentWebsiteId = websiteIdRef.current;
|
|
3070
|
+
if (currentWebsiteId && profile?.userId) {
|
|
3071
|
+
emitSocketEvent(socket, "tour:init", { websiteId: currentWebsiteId, userId: profile.userId, userType: profile.type });
|
|
3005
3072
|
}
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3073
|
+
};
|
|
3074
|
+
const handleServerState = (payload) => {
|
|
3075
|
+
if (typeof payload?.runId === "number") {
|
|
3076
|
+
runIdRef.current = payload.runId;
|
|
3077
|
+
}
|
|
3078
|
+
if (typeof payload?.turnId === "string" || payload?.turnId === null) {
|
|
3079
|
+
turnIdRef.current = payload.turnId ?? null;
|
|
3080
|
+
}
|
|
3081
|
+
setServerState(payload);
|
|
3082
|
+
};
|
|
3083
|
+
const handleCommandCancel = (payload) => {
|
|
3084
|
+
console.log("[TourClient] Received command_cancel:", payload);
|
|
3085
|
+
if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
|
|
3086
|
+
activeCommandBatchIdRef.current = null;
|
|
3087
|
+
activeExecutionTokenRef.current++;
|
|
3088
|
+
commandInFlightRef.current = false;
|
|
3089
|
+
setPlaybackState("idle");
|
|
3090
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3091
|
+
window.speechSynthesis.cancel();
|
|
3021
3092
|
}
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
if (
|
|
3093
|
+
}
|
|
3094
|
+
};
|
|
3095
|
+
const handleCommand = async (payload) => {
|
|
3096
|
+
const emitIfOpen = (ev, data) => {
|
|
3097
|
+
if (socketRef.current !== socket) return;
|
|
3098
|
+
emitSocketEvent(socket, ev, data);
|
|
3099
|
+
};
|
|
3100
|
+
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3101
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3102
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3103
|
+
if (voiceInputResolveRef.current) {
|
|
3104
|
+
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3105
|
+
voiceInputResolveRef.current = null;
|
|
3106
|
+
resolvePendingVoiceInput("");
|
|
3107
|
+
}
|
|
3108
|
+
setPlaybackState("executing");
|
|
3109
|
+
commandInFlightRef.current = true;
|
|
3110
|
+
const commandBatchId = payload.commandBatchId ?? null;
|
|
3111
|
+
turnIdRef.current = payload.turnId ?? turnIdRef.current;
|
|
3112
|
+
const clearCommandBatchId = () => {
|
|
3113
|
+
if (activeCommandBatchIdRef.current === commandBatchId) {
|
|
3027
3114
|
activeCommandBatchIdRef.current = null;
|
|
3028
|
-
activeExecutionTokenRef.current++;
|
|
3029
|
-
commandInFlightRef.current = false;
|
|
3030
|
-
setPlaybackState("idle");
|
|
3031
|
-
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3032
|
-
window.speechSynthesis.cancel();
|
|
3033
|
-
}
|
|
3034
|
-
}
|
|
3035
|
-
});
|
|
3036
|
-
socket.on("tour:command", async (payload) => {
|
|
3037
|
-
const emitIfOpen = (ev, data) => {
|
|
3038
|
-
if (socketRef.current !== socket) return;
|
|
3039
|
-
if (!socket.connected) return;
|
|
3040
|
-
socket.emit(ev, data);
|
|
3041
|
-
};
|
|
3042
|
-
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3043
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3044
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3045
|
-
if (voiceInputResolveRef.current) {
|
|
3046
|
-
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3047
|
-
voiceInputResolveRef.current = null;
|
|
3048
|
-
resolvePendingVoiceInput("");
|
|
3049
3115
|
}
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3116
|
+
};
|
|
3117
|
+
activeCommandBatchIdRef.current = commandBatchId;
|
|
3118
|
+
const executionToken = ++activeExecutionTokenRef.current;
|
|
3119
|
+
const activeTourId = tourRef.current?.id;
|
|
3120
|
+
const activePreviewRunId = previewRunIdRef.current;
|
|
3121
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3122
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3123
|
+
stepOrder: payload.stepIndex,
|
|
3124
|
+
eventType: "command_batch_received",
|
|
3125
|
+
payload: {
|
|
3126
|
+
commandBatchId,
|
|
3127
|
+
commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
|
|
3128
|
+
},
|
|
3129
|
+
currentStepOrder: payload.stepIndex
|
|
3130
|
+
});
|
|
3131
|
+
}
|
|
3132
|
+
if (typeof payload.stepIndex === "number") {
|
|
3133
|
+
const prevStep = stepIndexRef.current;
|
|
3134
|
+
stepIndexRef.current = payload.stepIndex;
|
|
3135
|
+
setCurrentStepIndex(payload.stepIndex);
|
|
3136
|
+
if (payload.stepIndex !== prevStep) {
|
|
3137
|
+
const tour = tourRef.current;
|
|
3138
|
+
const total = tour?.steps?.length ?? 0;
|
|
3139
|
+
if (tour && total > 0) {
|
|
3140
|
+
onStepChangeRef.current?.(payload.stepIndex, total, tour);
|
|
3057
3141
|
}
|
|
3058
|
-
}
|
|
3059
|
-
|
|
3060
|
-
const executionToken = ++activeExecutionTokenRef.current;
|
|
3061
|
-
const activeTourId = tourRef.current?.id;
|
|
3062
|
-
const activePreviewRunId = previewRunIdRef.current;
|
|
3063
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3142
|
+
}
|
|
3143
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3064
3144
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3065
3145
|
stepOrder: payload.stepIndex,
|
|
3066
|
-
eventType: "
|
|
3146
|
+
eventType: "step_started",
|
|
3067
3147
|
payload: {
|
|
3068
|
-
|
|
3069
|
-
|
|
3148
|
+
previousStepOrder: prevStep,
|
|
3149
|
+
stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
|
|
3070
3150
|
},
|
|
3071
3151
|
currentStepOrder: payload.stepIndex
|
|
3072
3152
|
});
|
|
3073
3153
|
}
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
}
|
|
3097
|
-
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3098
|
-
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3099
|
-
commandInFlightRef.current = false;
|
|
3100
|
-
emitIfOpen("tour:action_result", {
|
|
3101
|
-
success: false,
|
|
3102
|
-
reason: "invalid_commands",
|
|
3103
|
-
commandBatchId,
|
|
3104
|
-
runId: runIdRef.current,
|
|
3105
|
-
turnId: turnIdRef.current
|
|
3106
|
-
});
|
|
3107
|
-
clearCommandBatchId();
|
|
3108
|
-
return;
|
|
3154
|
+
}
|
|
3155
|
+
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3156
|
+
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3157
|
+
commandInFlightRef.current = false;
|
|
3158
|
+
emitIfOpen("tour:action_result", {
|
|
3159
|
+
success: false,
|
|
3160
|
+
reason: "invalid_commands",
|
|
3161
|
+
commandBatchId,
|
|
3162
|
+
runId: runIdRef.current,
|
|
3163
|
+
turnId: turnIdRef.current
|
|
3164
|
+
});
|
|
3165
|
+
clearCommandBatchId();
|
|
3166
|
+
return;
|
|
3167
|
+
}
|
|
3168
|
+
let shouldWait = false;
|
|
3169
|
+
const results = [];
|
|
3170
|
+
let batchPreferredWaitTarget = null;
|
|
3171
|
+
const assertNotInterrupted = () => {
|
|
3172
|
+
if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
|
|
3173
|
+
const error = new Error("interrupted");
|
|
3174
|
+
error.code = "INTERRUPTED";
|
|
3175
|
+
throw error;
|
|
3109
3176
|
}
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
}
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3125
|
-
onRetry: () => {
|
|
3126
|
-
assertNotInterrupted();
|
|
3127
|
-
},
|
|
3128
|
-
resolve: async () => {
|
|
3129
|
-
let targetEl = null;
|
|
3130
|
-
if (params.uid) {
|
|
3131
|
-
const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
|
|
3132
|
-
targetEl = getElementByUid(params.uid);
|
|
3133
|
-
}
|
|
3134
|
-
if (!targetEl) {
|
|
3135
|
-
targetEl = resolveElementFromHints({
|
|
3136
|
-
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3137
|
-
testId: params.testId ?? fallbackHints?.testId,
|
|
3138
|
-
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3139
|
-
}, fallbackStep, tagStore);
|
|
3140
|
-
}
|
|
3141
|
-
return targetEl;
|
|
3142
|
-
}
|
|
3143
|
-
});
|
|
3144
|
-
};
|
|
3145
|
-
const executeOne = async (action) => {
|
|
3146
|
-
assertNotInterrupted();
|
|
3147
|
-
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3148
|
-
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3149
|
-
const executeTimeline = async (params = {}) => {
|
|
3150
|
-
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3151
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
3152
|
-
assertNotInterrupted();
|
|
3153
|
-
const segment = segments[index];
|
|
3154
|
-
const segmentText = (segment?.text ?? "").trim();
|
|
3155
|
-
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3156
|
-
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3157
|
-
if (segmentDelayMs > 0) {
|
|
3158
|
-
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3159
|
-
}
|
|
3160
|
-
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3161
|
-
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3162
|
-
if (!gateTarget) {
|
|
3163
|
-
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3164
|
-
}
|
|
3165
|
-
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3166
|
-
}
|
|
3167
|
-
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3168
|
-
showCaption(segmentText);
|
|
3169
|
-
}
|
|
3170
|
-
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3171
|
-
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3172
|
-
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3173
|
-
onNearEnd: nextSegmentText ? () => {
|
|
3174
|
-
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3175
|
-
} : void 0
|
|
3176
|
-
}) : Promise.resolve();
|
|
3177
|
-
const eventsPromise = (async () => {
|
|
3178
|
-
for (const event of events) {
|
|
3179
|
-
assertNotInterrupted();
|
|
3180
|
-
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3181
|
-
if (delayMs > 0) {
|
|
3182
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3183
|
-
}
|
|
3184
|
-
if (event?.action) {
|
|
3185
|
-
await executeOne(event.action);
|
|
3186
|
-
}
|
|
3187
|
-
}
|
|
3188
|
-
})();
|
|
3189
|
-
await Promise.all([speechPromise, eventsPromise]);
|
|
3190
|
-
}
|
|
3191
|
-
if (params.removeHighlightAtEnd !== false) {
|
|
3192
|
-
removeHighlight();
|
|
3193
|
-
}
|
|
3194
|
-
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3195
|
-
removeCaption();
|
|
3196
|
-
}
|
|
3197
|
-
return !!params.waitForInput;
|
|
3198
|
-
};
|
|
3199
|
-
if (action.type === "speak") {
|
|
3200
|
-
const text = action.params?.text ?? "";
|
|
3201
|
-
if (!text.trim()) {
|
|
3202
|
-
return { result: null };
|
|
3177
|
+
};
|
|
3178
|
+
const resolveTargetElement2 = async (params = {}, fallbackStep) => {
|
|
3179
|
+
const fallbackHints = fallbackStep?.element ?? null;
|
|
3180
|
+
return retryLookup({
|
|
3181
|
+
timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
|
|
3182
|
+
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3183
|
+
onRetry: () => {
|
|
3184
|
+
assertNotInterrupted();
|
|
3185
|
+
},
|
|
3186
|
+
resolve: async () => {
|
|
3187
|
+
let targetEl = null;
|
|
3188
|
+
if (params.uid) {
|
|
3189
|
+
const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
|
|
3190
|
+
targetEl = getElementByUid(params.uid);
|
|
3203
3191
|
}
|
|
3204
|
-
if (
|
|
3205
|
-
|
|
3192
|
+
if (!targetEl) {
|
|
3193
|
+
targetEl = resolveElementFromHints({
|
|
3194
|
+
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3195
|
+
testId: params.testId ?? fallbackHints?.testId,
|
|
3196
|
+
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3197
|
+
}, fallbackStep, tagStore);
|
|
3206
3198
|
}
|
|
3207
|
-
|
|
3208
|
-
// Schedule narration immediately so the client keeps its speculative,
|
|
3209
|
-
// low-latency behavior, but defer batch completion until playback settles.
|
|
3210
|
-
waitForCompletion: false,
|
|
3211
|
-
interrupt: action.params?.interrupt
|
|
3212
|
-
});
|
|
3213
|
-
return { result: text, settlePromise };
|
|
3199
|
+
return targetEl;
|
|
3214
3200
|
}
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3201
|
+
});
|
|
3202
|
+
};
|
|
3203
|
+
const executeOne = async (action) => {
|
|
3204
|
+
assertNotInterrupted();
|
|
3205
|
+
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3206
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3207
|
+
const executeTimeline = async (params = {}) => {
|
|
3208
|
+
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3209
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
3210
|
+
assertNotInterrupted();
|
|
3211
|
+
const segment = segments[index];
|
|
3212
|
+
const segmentText = (segment?.text ?? "").trim();
|
|
3213
|
+
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3214
|
+
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3215
|
+
if (segmentDelayMs > 0) {
|
|
3216
|
+
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3217
|
+
}
|
|
3218
|
+
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3219
|
+
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3220
|
+
if (!gateTarget) {
|
|
3221
|
+
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3225
3222
|
}
|
|
3226
|
-
|
|
3227
|
-
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3228
|
-
return { result: "highlighted" };
|
|
3223
|
+
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3229
3224
|
}
|
|
3230
|
-
if (
|
|
3231
|
-
|
|
3232
|
-
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3233
|
-
);
|
|
3225
|
+
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3226
|
+
showCaption(segmentText);
|
|
3234
3227
|
}
|
|
3235
|
-
|
|
3228
|
+
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3229
|
+
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3230
|
+
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3231
|
+
onNearEnd: nextSegmentText ? () => {
|
|
3232
|
+
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3233
|
+
} : void 0
|
|
3234
|
+
}) : Promise.resolve();
|
|
3235
|
+
const eventsPromise = (async () => {
|
|
3236
|
+
for (const event of events) {
|
|
3237
|
+
assertNotInterrupted();
|
|
3238
|
+
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3239
|
+
if (delayMs > 0) {
|
|
3240
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3241
|
+
}
|
|
3242
|
+
if (event?.action) {
|
|
3243
|
+
await executeOne(event.action);
|
|
3244
|
+
}
|
|
3245
|
+
}
|
|
3246
|
+
})();
|
|
3247
|
+
await Promise.all([speechPromise, eventsPromise]);
|
|
3236
3248
|
}
|
|
3237
|
-
if (
|
|
3249
|
+
if (params.removeHighlightAtEnd !== false) {
|
|
3238
3250
|
removeHighlight();
|
|
3239
|
-
return { result: "highlight_removed" };
|
|
3240
3251
|
}
|
|
3241
|
-
if (
|
|
3242
|
-
|
|
3243
|
-
if (!targetEl) {
|
|
3244
|
-
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3245
|
-
throw new Error(
|
|
3246
|
-
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3247
|
-
);
|
|
3248
|
-
}
|
|
3249
|
-
removeHighlight();
|
|
3250
|
-
await performInteractiveClick(targetEl);
|
|
3251
|
-
const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3252
|
-
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3253
|
-
return { result: "clicked" };
|
|
3252
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3253
|
+
removeCaption();
|
|
3254
3254
|
}
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
}
|
|
3262
|
-
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3263
|
-
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3264
|
-
await performInteractiveFill(targetEl, value);
|
|
3265
|
-
return { result: value };
|
|
3255
|
+
return !!params.waitForInput;
|
|
3256
|
+
};
|
|
3257
|
+
if (action.type === "speak") {
|
|
3258
|
+
const text = action.params?.text ?? "";
|
|
3259
|
+
if (!text.trim()) {
|
|
3260
|
+
return { result: null };
|
|
3266
3261
|
}
|
|
3267
|
-
if (
|
|
3268
|
-
|
|
3269
|
-
const html2canvas2 = html2canvasModule.default;
|
|
3270
|
-
const canvas = await html2canvas2(document.body, {
|
|
3271
|
-
useCORS: true,
|
|
3272
|
-
allowTaint: true,
|
|
3273
|
-
scale: Math.min(window.devicePixelRatio, 2),
|
|
3274
|
-
width: window.innerWidth,
|
|
3275
|
-
height: window.innerHeight,
|
|
3276
|
-
x: window.scrollX,
|
|
3277
|
-
y: window.scrollY,
|
|
3278
|
-
logging: false
|
|
3279
|
-
});
|
|
3280
|
-
return { result: canvas.toDataURL("image/png") };
|
|
3262
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3263
|
+
showCaption(text);
|
|
3281
3264
|
}
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3265
|
+
const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
|
|
3266
|
+
// Schedule narration immediately so the client keeps its speculative,
|
|
3267
|
+
// low-latency behavior, but defer batch completion until playback settles.
|
|
3268
|
+
waitForCompletion: false,
|
|
3269
|
+
interrupt: action.params?.interrupt
|
|
3270
|
+
});
|
|
3271
|
+
return { result: text, settlePromise };
|
|
3272
|
+
}
|
|
3273
|
+
if (action.type === "play_timeline") {
|
|
3274
|
+
const timelineShouldWait = await executeTimeline(action.params);
|
|
3275
|
+
if (timelineShouldWait) shouldWait = true;
|
|
3276
|
+
return { result: "timeline_executed" };
|
|
3277
|
+
}
|
|
3278
|
+
if (action.type === "highlight_element") {
|
|
3279
|
+
const resolvedEl = await resolveTargetElement2(action.params, currentStep);
|
|
3280
|
+
if (resolvedEl) {
|
|
3281
|
+
if (isEditableWaitTarget(resolvedEl)) {
|
|
3282
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
|
|
3286
3283
|
}
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
return { result: nextUrl };
|
|
3284
|
+
showHighlight(resolvedEl, action.params?.label || action.label);
|
|
3285
|
+
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3286
|
+
return { result: "highlighted" };
|
|
3291
3287
|
}
|
|
3292
|
-
if (action.
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
}
|
|
3297
|
-
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3298
|
-
const res = await fetch(url, {
|
|
3299
|
-
method: "POST",
|
|
3300
|
-
headers: { "Content-Type": "application/json" },
|
|
3301
|
-
body: JSON.stringify({
|
|
3302
|
-
command: action.params?.command,
|
|
3303
|
-
socketId: agentSocketId
|
|
3304
|
-
})
|
|
3305
|
-
});
|
|
3306
|
-
if (!res.ok) {
|
|
3307
|
-
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3308
|
-
}
|
|
3309
|
-
if (action.params?.wait !== false) {
|
|
3310
|
-
await res.json();
|
|
3311
|
-
}
|
|
3312
|
-
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3313
|
-
await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3314
|
-
await syncAOM();
|
|
3315
|
-
return { result: action.params?.command ?? "executed" };
|
|
3288
|
+
if (!action.params?.optional) {
|
|
3289
|
+
throw new Error(
|
|
3290
|
+
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3291
|
+
);
|
|
3316
3292
|
}
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3293
|
+
return { result: "highlight_optional_miss" };
|
|
3294
|
+
}
|
|
3295
|
+
if (action.type === "remove_highlight") {
|
|
3296
|
+
removeHighlight();
|
|
3297
|
+
return { result: "highlight_removed" };
|
|
3298
|
+
}
|
|
3299
|
+
if (action.type === "click_element") {
|
|
3300
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3301
|
+
if (!targetEl) {
|
|
3302
|
+
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3303
|
+
throw new Error(
|
|
3304
|
+
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3305
|
+
);
|
|
3325
3306
|
}
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3307
|
+
removeHighlight();
|
|
3308
|
+
await performInteractiveClick(targetEl);
|
|
3309
|
+
const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3310
|
+
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3311
|
+
return { result: "clicked" };
|
|
3312
|
+
}
|
|
3313
|
+
if (action.type === "fill_input") {
|
|
3314
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3315
|
+
if (!targetEl) {
|
|
3316
|
+
throw new Error(
|
|
3317
|
+
`fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3318
|
+
);
|
|
3329
3319
|
}
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3320
|
+
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3321
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3322
|
+
await performInteractiveFill(targetEl, value);
|
|
3323
|
+
return { result: value };
|
|
3324
|
+
}
|
|
3325
|
+
if (action.type === "take_screenshot") {
|
|
3326
|
+
const html2canvasModule = await import("html2canvas");
|
|
3327
|
+
const html2canvas2 = html2canvasModule.default;
|
|
3328
|
+
const canvas = await html2canvas2(document.body, {
|
|
3329
|
+
useCORS: true,
|
|
3330
|
+
allowTaint: true,
|
|
3331
|
+
scale: Math.min(window.devicePixelRatio, 2),
|
|
3332
|
+
width: window.innerWidth,
|
|
3333
|
+
height: window.innerHeight,
|
|
3334
|
+
x: window.scrollX,
|
|
3335
|
+
y: window.scrollY,
|
|
3336
|
+
logging: false
|
|
3337
|
+
});
|
|
3338
|
+
return { result: canvas.toDataURL("image/png") };
|
|
3339
|
+
}
|
|
3340
|
+
if (action.type === "navigate_to_url") {
|
|
3341
|
+
const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
|
|
3342
|
+
if (!nextUrl) {
|
|
3343
|
+
throw new Error("navigate_to_url missing url");
|
|
3333
3344
|
}
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
const isTerminal = isTerminalAction(command);
|
|
3344
|
-
if (isTerminal) {
|
|
3345
|
-
await Promise.all(pendingUIActions);
|
|
3346
|
-
pendingUIActions.length = 0;
|
|
3347
|
-
}
|
|
3348
|
-
const executionPromise = (async () => {
|
|
3349
|
-
const execution = await executeOne(command);
|
|
3350
|
-
await execution.settlePromise;
|
|
3351
|
-
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3352
|
-
})();
|
|
3353
|
-
if (isTerminal) {
|
|
3354
|
-
await executionPromise;
|
|
3355
|
-
} else {
|
|
3356
|
-
pendingUIActions.push(executionPromise);
|
|
3357
|
-
}
|
|
3345
|
+
await navigateToTourUrl(nextUrl);
|
|
3346
|
+
const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3347
|
+
await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3348
|
+
return { result: nextUrl };
|
|
3349
|
+
}
|
|
3350
|
+
if (action.type === "execute_agent_action") {
|
|
3351
|
+
const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
|
|
3352
|
+
if (!agentSocketId) {
|
|
3353
|
+
throw new Error("No socketId available for execute_agent_action");
|
|
3358
3354
|
}
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3355
|
+
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3356
|
+
const res = await fetch(url, {
|
|
3357
|
+
method: "POST",
|
|
3358
|
+
headers: { "Content-Type": "application/json" },
|
|
3359
|
+
body: JSON.stringify({
|
|
3360
|
+
command: action.params?.command,
|
|
3361
|
+
socketId: agentSocketId
|
|
3362
|
+
})
|
|
3362
3363
|
});
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
commandInFlightRef.current = false;
|
|
3366
|
-
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3367
|
-
if (interrupted) {
|
|
3368
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3369
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3370
|
-
stepOrder: stepIndexRef.current,
|
|
3371
|
-
eventType: "command_batch_interrupted",
|
|
3372
|
-
payload: {
|
|
3373
|
-
commandBatchId,
|
|
3374
|
-
partialResults: results
|
|
3375
|
-
},
|
|
3376
|
-
currentStepOrder: stepIndexRef.current
|
|
3377
|
-
});
|
|
3378
|
-
}
|
|
3379
|
-
emitIfOpen("tour:action_result", {
|
|
3380
|
-
success: true,
|
|
3381
|
-
interrupted: true,
|
|
3382
|
-
results,
|
|
3383
|
-
commandBatchId,
|
|
3384
|
-
runId: runIdRef.current,
|
|
3385
|
-
turnId: turnIdRef.current
|
|
3386
|
-
});
|
|
3387
|
-
clearCommandBatchId();
|
|
3388
|
-
return;
|
|
3364
|
+
if (!res.ok) {
|
|
3365
|
+
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3389
3366
|
}
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3393
|
-
stepOrder: stepIndexRef.current,
|
|
3394
|
-
eventType: "command_batch_failed",
|
|
3395
|
-
payload: {
|
|
3396
|
-
commandBatchId,
|
|
3397
|
-
error: String(err),
|
|
3398
|
-
partialResults: results
|
|
3399
|
-
},
|
|
3400
|
-
status: "active",
|
|
3401
|
-
currentStepOrder: stepIndexRef.current
|
|
3402
|
-
});
|
|
3367
|
+
if (action.params?.wait !== false) {
|
|
3368
|
+
await res.json();
|
|
3403
3369
|
}
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
results,
|
|
3409
|
-
commandBatchId,
|
|
3410
|
-
runId: runIdRef.current,
|
|
3411
|
-
turnId: turnIdRef.current
|
|
3412
|
-
});
|
|
3413
|
-
clearCommandBatchId();
|
|
3414
|
-
return;
|
|
3370
|
+
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3371
|
+
await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3372
|
+
await syncAOM();
|
|
3373
|
+
return { result: action.params?.command ?? "executed" };
|
|
3415
3374
|
}
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3421
|
-
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3422
|
-
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3423
|
-
let manualWaitPromise = null;
|
|
3424
|
-
let manualWaitKind = null;
|
|
3425
|
-
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3426
|
-
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3427
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3428
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3429
|
-
if (waitTargetHints) {
|
|
3430
|
-
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3431
|
-
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3432
|
-
manualWaitTarget = preferredWaitTarget;
|
|
3433
|
-
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3434
|
-
}
|
|
3435
|
-
if (manualWaitTarget) {
|
|
3436
|
-
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3437
|
-
manualWaitPromise = manualWait.promise;
|
|
3438
|
-
manualWaitKind = manualWait.kind;
|
|
3439
|
-
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3440
|
-
}
|
|
3375
|
+
if (action.type === "wait_for_user_action") {
|
|
3376
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3377
|
+
if (!targetEl) {
|
|
3378
|
+
throw new Error("wait_for_user_action target not found");
|
|
3441
3379
|
}
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3380
|
+
const eventName = action.params?.event ?? "click";
|
|
3381
|
+
await waitForUserAction(targetEl, eventName);
|
|
3382
|
+
return { result: `waited_for_${eventName}` };
|
|
3383
|
+
}
|
|
3384
|
+
if (action.type === "wait_for_input") {
|
|
3385
|
+
shouldWait = true;
|
|
3386
|
+
return { result: "waiting_for_input" };
|
|
3387
|
+
}
|
|
3388
|
+
if (action.type === "end_tour") {
|
|
3389
|
+
handleTourEnd();
|
|
3390
|
+
return { result: "ended" };
|
|
3391
|
+
}
|
|
3392
|
+
console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
|
|
3393
|
+
return { result: "unknown_action_skipped" };
|
|
3394
|
+
};
|
|
3395
|
+
try {
|
|
3396
|
+
const resultsBuffer = new Array(payload.commands.length);
|
|
3397
|
+
const pendingUIActions = [];
|
|
3398
|
+
for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
|
|
3399
|
+
const command = payload.commands[commandIndex];
|
|
3400
|
+
assertNotInterrupted();
|
|
3401
|
+
const isTerminal = isTerminalAction(command);
|
|
3402
|
+
if (isTerminal) {
|
|
3403
|
+
await Promise.all(pendingUIActions);
|
|
3404
|
+
pendingUIActions.length = 0;
|
|
3448
3405
|
}
|
|
3449
|
-
|
|
3450
|
-
const
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3459
|
-
}
|
|
3406
|
+
const executionPromise = (async () => {
|
|
3407
|
+
const execution = await executeOne(command);
|
|
3408
|
+
await execution.settlePromise;
|
|
3409
|
+
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3410
|
+
})();
|
|
3411
|
+
if (isTerminal) {
|
|
3412
|
+
await executionPromise;
|
|
3413
|
+
} else {
|
|
3414
|
+
pendingUIActions.push(executionPromise);
|
|
3460
3415
|
}
|
|
3461
|
-
|
|
3416
|
+
}
|
|
3417
|
+
await Promise.all(pendingUIActions);
|
|
3418
|
+
resultsBuffer.forEach((res) => {
|
|
3419
|
+
if (res) results.push(res);
|
|
3420
|
+
});
|
|
3421
|
+
await syncAOM();
|
|
3422
|
+
} catch (err) {
|
|
3423
|
+
commandInFlightRef.current = false;
|
|
3424
|
+
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3425
|
+
if (interrupted) {
|
|
3462
3426
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3463
3427
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3464
3428
|
stepOrder: stepIndexRef.current,
|
|
3465
|
-
eventType: "
|
|
3429
|
+
eventType: "command_batch_interrupted",
|
|
3466
3430
|
payload: {
|
|
3467
3431
|
commandBatchId,
|
|
3468
|
-
results
|
|
3432
|
+
partialResults: results
|
|
3469
3433
|
},
|
|
3470
3434
|
currentStepOrder: stepIndexRef.current
|
|
3471
3435
|
});
|
|
3472
3436
|
}
|
|
3473
3437
|
emitIfOpen("tour:action_result", {
|
|
3474
3438
|
success: true,
|
|
3475
|
-
|
|
3439
|
+
interrupted: true,
|
|
3476
3440
|
results,
|
|
3477
3441
|
commandBatchId,
|
|
3478
3442
|
runId: runIdRef.current,
|
|
3479
3443
|
turnId: turnIdRef.current
|
|
3480
3444
|
});
|
|
3481
3445
|
clearCommandBatchId();
|
|
3482
|
-
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3483
|
-
if (pendingInputBufRef.current) {
|
|
3484
|
-
const flushed = pendingInputBufRef.current;
|
|
3485
|
-
pendingInputBufRef.current = null;
|
|
3486
|
-
resolve(flushed);
|
|
3487
|
-
return;
|
|
3488
|
-
}
|
|
3489
|
-
voiceInputResolveRef.current = (text) => {
|
|
3490
|
-
voiceInputResolveRef.current = null;
|
|
3491
|
-
resolve(text);
|
|
3492
|
-
};
|
|
3493
|
-
});
|
|
3494
|
-
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3495
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3496
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3497
|
-
voiceInputResolveRef.current = null;
|
|
3498
|
-
setPlaybackState("executing");
|
|
3499
|
-
const transcript = userText.trim();
|
|
3500
|
-
if (!transcript) {
|
|
3501
|
-
return;
|
|
3502
|
-
}
|
|
3503
|
-
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3504
|
-
await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
|
|
3505
|
-
await syncAOM();
|
|
3506
|
-
emitIfOpen("tour:user_input", {
|
|
3507
|
-
transcript,
|
|
3508
|
-
runId: runIdRef.current,
|
|
3509
|
-
turnId: turnIdRef.current
|
|
3510
|
-
});
|
|
3511
|
-
});
|
|
3512
3446
|
return;
|
|
3513
3447
|
}
|
|
3448
|
+
console.error("[TourClient] Command batch execution failed:", err);
|
|
3514
3449
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3515
3450
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3516
3451
|
stepOrder: stepIndexRef.current,
|
|
3517
|
-
eventType: "
|
|
3452
|
+
eventType: "command_batch_failed",
|
|
3518
3453
|
payload: {
|
|
3519
3454
|
commandBatchId,
|
|
3520
|
-
|
|
3455
|
+
error: String(err),
|
|
3456
|
+
partialResults: results
|
|
3521
3457
|
},
|
|
3458
|
+
status: "active",
|
|
3522
3459
|
currentStepOrder: stepIndexRef.current
|
|
3523
3460
|
});
|
|
3524
3461
|
}
|
|
3525
3462
|
emitIfOpen("tour:action_result", {
|
|
3526
|
-
success:
|
|
3463
|
+
success: false,
|
|
3464
|
+
reason: "execution_error",
|
|
3465
|
+
error: String(err),
|
|
3527
3466
|
results,
|
|
3528
3467
|
commandBatchId,
|
|
3529
3468
|
runId: runIdRef.current,
|
|
3530
3469
|
turnId: turnIdRef.current
|
|
3531
3470
|
});
|
|
3532
3471
|
clearCommandBatchId();
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
const
|
|
3538
|
-
const
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3472
|
+
return;
|
|
3473
|
+
}
|
|
3474
|
+
commandInFlightRef.current = false;
|
|
3475
|
+
if (shouldWait && !skipRequestedRef.current) {
|
|
3476
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3477
|
+
const waitCondition = currentStep?.onboarding?.waitCondition;
|
|
3478
|
+
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3479
|
+
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3480
|
+
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3481
|
+
let manualWaitPromise = null;
|
|
3482
|
+
let manualWaitKind = null;
|
|
3483
|
+
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3484
|
+
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3485
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3486
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3487
|
+
if (waitTargetHints) {
|
|
3488
|
+
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3489
|
+
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3490
|
+
manualWaitTarget = preferredWaitTarget;
|
|
3491
|
+
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3492
|
+
}
|
|
3493
|
+
if (manualWaitTarget) {
|
|
3494
|
+
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3495
|
+
manualWaitPromise = manualWait.promise;
|
|
3496
|
+
manualWaitKind = manualWait.kind;
|
|
3497
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3498
|
+
}
|
|
3499
|
+
}
|
|
3500
|
+
if (!manualWaitPromise && preferredWaitTarget) {
|
|
3501
|
+
const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
|
|
3502
|
+
manualWaitPromise = manualWait.promise;
|
|
3503
|
+
manualWaitKind = manualWait.kind;
|
|
3504
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3505
|
+
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3542
3506
|
}
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3507
|
+
if (!manualWaitPromise && inputLikeWait) {
|
|
3508
|
+
const firstInput = document.querySelector(
|
|
3509
|
+
'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
|
|
3510
|
+
);
|
|
3511
|
+
if (firstInput) {
|
|
3512
|
+
const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
|
|
3513
|
+
manualWaitPromise = manualWait.promise;
|
|
3514
|
+
manualWaitKind = manualWait.kind;
|
|
3515
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3516
|
+
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3517
|
+
}
|
|
3518
|
+
}
|
|
3519
|
+
setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
|
|
3520
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3521
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3522
|
+
stepOrder: stepIndexRef.current,
|
|
3523
|
+
eventType: "waiting_for_input",
|
|
3557
3524
|
payload: {
|
|
3558
|
-
|
|
3559
|
-
|
|
3525
|
+
commandBatchId,
|
|
3526
|
+
results
|
|
3560
3527
|
},
|
|
3561
|
-
currentStepOrder:
|
|
3528
|
+
currentStepOrder: stepIndexRef.current
|
|
3562
3529
|
});
|
|
3563
3530
|
}
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3531
|
+
emitIfOpen("tour:action_result", {
|
|
3532
|
+
success: true,
|
|
3533
|
+
waitingForInput: true,
|
|
3534
|
+
results,
|
|
3535
|
+
commandBatchId,
|
|
3536
|
+
runId: runIdRef.current,
|
|
3537
|
+
turnId: turnIdRef.current
|
|
3538
|
+
});
|
|
3539
|
+
clearCommandBatchId();
|
|
3540
|
+
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3541
|
+
if (pendingInputBufRef.current) {
|
|
3542
|
+
const flushed = pendingInputBufRef.current;
|
|
3543
|
+
pendingInputBufRef.current = null;
|
|
3544
|
+
resolve(flushed);
|
|
3545
|
+
return;
|
|
3573
3546
|
}
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3547
|
+
voiceInputResolveRef.current = (text) => {
|
|
3548
|
+
voiceInputResolveRef.current = null;
|
|
3549
|
+
resolve(text);
|
|
3550
|
+
};
|
|
3551
|
+
});
|
|
3552
|
+
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3553
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3554
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3555
|
+
voiceInputResolveRef.current = null;
|
|
3556
|
+
setPlaybackState("executing");
|
|
3557
|
+
const transcript = userText.trim();
|
|
3558
|
+
if (!transcript) {
|
|
3559
|
+
return;
|
|
3560
|
+
}
|
|
3561
|
+
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3562
|
+
await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
|
|
3563
|
+
await syncAOM();
|
|
3564
|
+
emitIfOpen("tour:user_input", {
|
|
3565
|
+
transcript,
|
|
3566
|
+
runId: runIdRef.current,
|
|
3567
|
+
turnId: turnIdRef.current
|
|
3568
|
+
});
|
|
3569
|
+
});
|
|
3570
|
+
return;
|
|
3571
|
+
}
|
|
3572
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3573
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3574
|
+
stepOrder: stepIndexRef.current,
|
|
3575
|
+
eventType: "command_batch_completed",
|
|
3576
|
+
payload: {
|
|
3577
|
+
commandBatchId,
|
|
3578
|
+
results
|
|
3579
|
+
},
|
|
3580
|
+
currentStepOrder: stepIndexRef.current
|
|
3581
|
+
});
|
|
3582
|
+
}
|
|
3583
|
+
emitIfOpen("tour:action_result", {
|
|
3584
|
+
success: true,
|
|
3585
|
+
results,
|
|
3586
|
+
commandBatchId,
|
|
3587
|
+
runId: runIdRef.current,
|
|
3588
|
+
turnId: turnIdRef.current
|
|
3577
3589
|
});
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3590
|
+
clearCommandBatchId();
|
|
3591
|
+
};
|
|
3592
|
+
const handleTourStart = async (tourData) => {
|
|
3593
|
+
if (isActiveRef.current) return;
|
|
3594
|
+
runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
|
|
3595
|
+
const tour = tourData.tourContext ?? tourRef.current;
|
|
3596
|
+
const expType = experienceTypeRef.current;
|
|
3597
|
+
if (tour?.type && tour.type !== expType) {
|
|
3598
|
+
console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
|
|
3599
|
+
return;
|
|
3600
|
+
}
|
|
3601
|
+
skipRequestedRef.current = false;
|
|
3602
|
+
const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
|
|
3603
|
+
isActiveRef.current = true;
|
|
3604
|
+
setIsActive(true);
|
|
3605
|
+
setActiveTour(tour ?? null);
|
|
3606
|
+
tourRef.current = tour ?? null;
|
|
3607
|
+
setTotalSteps(total);
|
|
3608
|
+
stepIndexRef.current = 0;
|
|
3609
|
+
setCurrentStepIndex(0);
|
|
3610
|
+
setPlaybackState("intro");
|
|
3611
|
+
if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
|
|
3612
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
|
|
3613
|
+
stepOrder: 0,
|
|
3614
|
+
eventType: "tour_started",
|
|
3615
|
+
payload: {
|
|
3616
|
+
totalSteps: total,
|
|
3617
|
+
source: "sdk_test_preview"
|
|
3618
|
+
},
|
|
3619
|
+
currentStepOrder: 0
|
|
3620
|
+
});
|
|
3621
|
+
}
|
|
3622
|
+
try {
|
|
3623
|
+
const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
|
|
3624
|
+
const aom = generateMinifiedAOM2();
|
|
3625
|
+
if (socketRef.current === socket) {
|
|
3626
|
+
emitSocketEvent(socket, "tour:sync_dom", {
|
|
3627
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3628
|
+
aom: aom.nodes,
|
|
3629
|
+
domSummary: captureDomSummary()
|
|
3630
|
+
});
|
|
3582
3631
|
}
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3632
|
+
} catch (e) {
|
|
3633
|
+
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3634
|
+
}
|
|
3635
|
+
};
|
|
3636
|
+
const handleTourUpdate = (payload) => {
|
|
3637
|
+
const updatedTour = payload?.tourContext;
|
|
3638
|
+
if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
|
|
3639
|
+
return;
|
|
3640
|
+
}
|
|
3641
|
+
tourRef.current = updatedTour;
|
|
3642
|
+
setActiveTour(updatedTour);
|
|
3643
|
+
const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
|
|
3644
|
+
setTotalSteps(nextTotal);
|
|
3645
|
+
if (typeof payload.currentStepIndex === "number") {
|
|
3646
|
+
const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
|
|
3647
|
+
stepIndexRef.current = clampedStepIndex;
|
|
3648
|
+
setCurrentStepIndex(clampedStepIndex);
|
|
3649
|
+
if (nextTotal > 0) {
|
|
3650
|
+
onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
|
|
3594
3651
|
}
|
|
3595
|
-
}
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
}
|
|
3608
|
-
}
|
|
3652
|
+
}
|
|
3653
|
+
};
|
|
3654
|
+
const handleTourEndEvent = () => {
|
|
3655
|
+
setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
|
|
3656
|
+
handleTourEnd();
|
|
3657
|
+
};
|
|
3658
|
+
const handleDebugLog = (entry) => {
|
|
3659
|
+
const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
|
|
3660
|
+
if (isDev) {
|
|
3661
|
+
console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
|
|
3662
|
+
if (typeof window !== "undefined") {
|
|
3663
|
+
window.dispatchEvent(new CustomEvent("modelnex-debug", {
|
|
3664
|
+
detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
|
|
3665
|
+
}));
|
|
3609
3666
|
}
|
|
3610
|
-
});
|
|
3611
|
-
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3612
|
-
});
|
|
3613
|
-
return () => {
|
|
3614
|
-
cancelled = true;
|
|
3615
|
-
const toClose = createdSocket ?? socketRef.current;
|
|
3616
|
-
if (toClose) {
|
|
3617
|
-
toClose.disconnect();
|
|
3618
3667
|
}
|
|
3619
|
-
|
|
3668
|
+
};
|
|
3669
|
+
socket.on("connect", handleConnect);
|
|
3670
|
+
socket.on("tour:server_state", handleServerState);
|
|
3671
|
+
socket.on("tour:command_cancel", handleCommandCancel);
|
|
3672
|
+
socket.on("tour:command", handleCommand);
|
|
3673
|
+
socket.on("tour:start", handleTourStart);
|
|
3674
|
+
socket.on("tour:update", handleTourUpdate);
|
|
3675
|
+
socket.on("tour:end", handleTourEndEvent);
|
|
3676
|
+
socket.on("tour:debug_log", handleDebugLog);
|
|
3677
|
+
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3678
|
+
return () => {
|
|
3679
|
+
socket.off("connect", handleConnect);
|
|
3680
|
+
socket.off("tour:server_state", handleServerState);
|
|
3681
|
+
socket.off("tour:command_cancel", handleCommandCancel);
|
|
3682
|
+
socket.off("tour:command", handleCommand);
|
|
3683
|
+
socket.off("tour:start", handleTourStart);
|
|
3684
|
+
socket.off("tour:update", handleTourUpdate);
|
|
3685
|
+
socket.off("tour:end", handleTourEndEvent);
|
|
3686
|
+
socket.off("tour:debug_log", handleDebugLog);
|
|
3687
|
+
const toClose = socket;
|
|
3620
3688
|
socketRef.current = null;
|
|
3621
3689
|
setServerState(null);
|
|
3622
3690
|
runIdRef.current = null;
|
|
3623
3691
|
turnIdRef.current = null;
|
|
3692
|
+
tourSocketPool.release(serverUrl, toClose);
|
|
3624
3693
|
};
|
|
3625
|
-
}, [serverUrl,
|
|
3694
|
+
}, [serverUrl, disabled]);
|
|
3626
3695
|
useEffect11(() => {
|
|
3627
3696
|
if (disabled) return;
|
|
3628
3697
|
const s = socketRef.current;
|
|
3629
3698
|
const profile = userProfile;
|
|
3630
|
-
if (!
|
|
3631
|
-
|
|
3699
|
+
if (!websiteId || !profile?.userId) return;
|
|
3700
|
+
const timer = setTimeout(() => {
|
|
3701
|
+
emitSocketEvent(s, "tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3702
|
+
}, 150);
|
|
3703
|
+
return () => clearTimeout(timer);
|
|
3632
3704
|
}, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
|
|
3633
3705
|
useEffect11(() => {
|
|
3634
3706
|
if (!showCaptions || !isReviewMode) {
|
|
@@ -3636,8 +3708,8 @@ function useTourPlayback({
|
|
|
3636
3708
|
}
|
|
3637
3709
|
}, [showCaptions, isReviewMode]);
|
|
3638
3710
|
useEffect11(() => {
|
|
3639
|
-
if (!
|
|
3640
|
-
socketRef.current
|
|
3711
|
+
if (!isActiveRef.current) return;
|
|
3712
|
+
emitSocketEvent(socketRef.current, "tour:client_state", {
|
|
3641
3713
|
runId: runIdRef.current,
|
|
3642
3714
|
turnId: turnIdRef.current,
|
|
3643
3715
|
commandBatchId: activeCommandBatchIdRef.current,
|
|
@@ -3650,17 +3722,17 @@ function useTourPlayback({
|
|
|
3650
3722
|
});
|
|
3651
3723
|
}, [isActive, playbackState, voice.isListening, voice.isSpeaking]);
|
|
3652
3724
|
const syncAOM = useCallback7(async () => {
|
|
3653
|
-
if (!
|
|
3725
|
+
if (!isActiveRef.current) return;
|
|
3654
3726
|
const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
|
|
3655
3727
|
const aom = generateMinifiedAOM2();
|
|
3656
|
-
socketRef.current
|
|
3728
|
+
emitSocketEvent(socketRef.current, "tour:sync_dom", {
|
|
3657
3729
|
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3658
3730
|
aom: aom.nodes,
|
|
3659
3731
|
domSummary: captureDomSummary()
|
|
3660
3732
|
});
|
|
3661
3733
|
}, []);
|
|
3662
3734
|
const interruptExecution = useCallback7((transcript) => {
|
|
3663
|
-
if (!socketRef.current
|
|
3735
|
+
if (!isSocketWritable(socketRef.current) || !isActiveRef.current) return false;
|
|
3664
3736
|
if (!commandInFlightRef.current && !voice.isSpeaking) return false;
|
|
3665
3737
|
interruptedForQuestionRef.current = true;
|
|
3666
3738
|
activeExecutionTokenRef.current += 1;
|
|
@@ -3668,21 +3740,22 @@ function useTourPlayback({
|
|
|
3668
3740
|
removeHighlight();
|
|
3669
3741
|
removeCaption();
|
|
3670
3742
|
voice.stopSpeaking();
|
|
3671
|
-
socketRef.current
|
|
3743
|
+
if (emitSocketEvent(socketRef.current, "tour:action_result", {
|
|
3672
3744
|
success: true,
|
|
3673
3745
|
interrupted: true,
|
|
3674
3746
|
transcript,
|
|
3675
3747
|
commandBatchId: activeCommandBatchIdRef.current,
|
|
3676
3748
|
runId: runIdRef.current,
|
|
3677
3749
|
turnId: turnIdRef.current
|
|
3678
|
-
})
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3750
|
+
})) {
|
|
3751
|
+
activeCommandBatchIdRef.current = null;
|
|
3752
|
+
emitSocketEvent(socketRef.current, "tour:user_input", {
|
|
3753
|
+
transcript,
|
|
3754
|
+
interrupted: true,
|
|
3755
|
+
runId: runIdRef.current,
|
|
3756
|
+
turnId: turnIdRef.current
|
|
3757
|
+
});
|
|
3758
|
+
}
|
|
3686
3759
|
setPlaybackState("thinking");
|
|
3687
3760
|
return true;
|
|
3688
3761
|
}, [voice]);
|
|
@@ -3698,9 +3771,7 @@ function useTourPlayback({
|
|
|
3698
3771
|
removeCaption();
|
|
3699
3772
|
voice.stopSpeaking();
|
|
3700
3773
|
voice.stopListening();
|
|
3701
|
-
|
|
3702
|
-
socketRef.current.emit("tour:abort");
|
|
3703
|
-
}
|
|
3774
|
+
emitSocketEvent(socketRef.current, "tour:abort");
|
|
3704
3775
|
if (reviewModeRef.current && tourRef.current?.id && previewRunIdRef.current) {
|
|
3705
3776
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tourRef.current.id, previewRunIdRef.current, websiteId, {
|
|
3706
3777
|
stepOrder: stepIndexRef.current,
|
|
@@ -3772,7 +3843,7 @@ function useTourPlayback({
|
|
|
3772
3843
|
await new Promise((r) => setTimeout(r, 200));
|
|
3773
3844
|
retries++;
|
|
3774
3845
|
}
|
|
3775
|
-
if (!socketRef.current
|
|
3846
|
+
if (!isSocketWritable(socketRef.current)) {
|
|
3776
3847
|
console.warn("[TourClient] Cannot run tour, socket not connected.");
|
|
3777
3848
|
return;
|
|
3778
3849
|
}
|
|
@@ -3800,7 +3871,7 @@ function useTourPlayback({
|
|
|
3800
3871
|
previewRunIdRef.current = null;
|
|
3801
3872
|
}
|
|
3802
3873
|
tourRef.current = tour;
|
|
3803
|
-
socketRef.current
|
|
3874
|
+
emitSocketEvent(socketRef.current, "tour:request_start", {
|
|
3804
3875
|
tourId: tour.id,
|
|
3805
3876
|
previewRunId: previewRunIdRef.current,
|
|
3806
3877
|
tourContext: tour
|
|
@@ -3948,8 +4019,8 @@ function useTourPlayback({
|
|
|
3948
4019
|
const revisionVersion = Number(response?.revision?.versionNumber);
|
|
3949
4020
|
const appliedMessage = Number.isFinite(revisionVersion) && revisionVersion > 0 ? `Applied to the draft as version ${revisionVersion}.` : "Correction applied to the draft.";
|
|
3950
4021
|
setReviewStatusMessage(apply ? appliedMessage : "Correction saved for review.");
|
|
3951
|
-
if (apply && playbackState === "paused" &&
|
|
3952
|
-
socketRef.current
|
|
4022
|
+
if (apply && playbackState === "paused" && isActiveRef.current) {
|
|
4023
|
+
emitSocketEvent(socketRef.current, "tour:resume");
|
|
3953
4024
|
setPlaybackState("executing");
|
|
3954
4025
|
}
|
|
3955
4026
|
} catch (err) {
|
|
@@ -3973,14 +4044,14 @@ function useTourPlayback({
|
|
|
3973
4044
|
stopTour();
|
|
3974
4045
|
}, [stopTour]);
|
|
3975
4046
|
const pauseTour = useCallback7(() => {
|
|
3976
|
-
if (socketRef.current
|
|
3977
|
-
socketRef.current
|
|
4047
|
+
if (isSocketWritable(socketRef.current) && isActiveRef.current) {
|
|
4048
|
+
emitSocketEvent(socketRef.current, "tour:pause");
|
|
3978
4049
|
setPlaybackState("paused");
|
|
3979
4050
|
}
|
|
3980
4051
|
}, []);
|
|
3981
4052
|
const resumeTour = useCallback7(() => {
|
|
3982
|
-
if (socketRef.current
|
|
3983
|
-
socketRef.current
|
|
4053
|
+
if (isSocketWritable(socketRef.current) && isActiveRef.current) {
|
|
4054
|
+
emitSocketEvent(socketRef.current, "tour:resume");
|
|
3984
4055
|
setPlaybackState("executing");
|
|
3985
4056
|
}
|
|
3986
4057
|
}, []);
|
|
@@ -4002,9 +4073,9 @@ function useTourPlayback({
|
|
|
4002
4073
|
if (voiceInputResolveRef.current) {
|
|
4003
4074
|
console.log("[TourAgent] Resolving loop waiting_voice with text:", text);
|
|
4004
4075
|
voiceInputResolveRef.current(text);
|
|
4005
|
-
} else if (socketRef.current
|
|
4076
|
+
} else if (isSocketWritable(socketRef.current)) {
|
|
4006
4077
|
console.log("[TourAgent] Forwarding ambient voice to server:", text);
|
|
4007
|
-
socketRef.current
|
|
4078
|
+
emitSocketEvent(socketRef.current, "tour:user_input", {
|
|
4008
4079
|
transcript: text,
|
|
4009
4080
|
runId: runIdRef.current,
|
|
4010
4081
|
turnId: turnIdRef.current
|
|
@@ -5252,7 +5323,7 @@ function useVoice(serverUrl) {
|
|
|
5252
5323
|
(async () => {
|
|
5253
5324
|
try {
|
|
5254
5325
|
const ioModule = await import("socket.io-client");
|
|
5255
|
-
const
|
|
5326
|
+
const io3 = ioModule.default || ioModule;
|
|
5256
5327
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
5257
5328
|
audio: {
|
|
5258
5329
|
echoCancellation: true,
|
|
@@ -5274,7 +5345,7 @@ function useVoice(serverUrl) {
|
|
|
5274
5345
|
audioBitsPerSecond: 128e3
|
|
5275
5346
|
});
|
|
5276
5347
|
mediaRecorderRef.current = recorder;
|
|
5277
|
-
const socket =
|
|
5348
|
+
const socket = io3(serverUrl, {
|
|
5278
5349
|
path: "/socket.io",
|
|
5279
5350
|
transports: resolveSocketIoTransports(serverUrl, "websocket-first")
|
|
5280
5351
|
});
|
|
@@ -10794,7 +10865,7 @@ var ModelNexProvider = ({
|
|
|
10794
10865
|
socketId,
|
|
10795
10866
|
devMode
|
|
10796
10867
|
}),
|
|
10797
|
-
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
|
|
10868
|
+
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
|
|
10798
10869
|
);
|
|
10799
10870
|
return React8.createElement(
|
|
10800
10871
|
ModelNexContext.Provider,
|