@modelnex/sdk 0.5.15 → 0.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +570 -565
- package/dist/index.mjs +570 -565
- package/package.json +11 -12
- package/dist/aom-GA6W42DG.mjs +0 -71
- package/dist/aom-J6NYMGDW.mjs +0 -69
- package/dist/chunk-6DZX6EAA.mjs +0 -37
- package/dist/dom-sync-Y7Z7TOU6.mjs +0 -57
package/dist/index.mjs
CHANGED
|
@@ -2416,6 +2416,7 @@ function isTourEligible(tour, userProfile) {
|
|
|
2416
2416
|
|
|
2417
2417
|
// src/hooks/useTourPlayback.ts
|
|
2418
2418
|
import { useState as useState7, useRef as useRef8, useCallback as useCallback7, useEffect as useEffect11, useContext as useContext4 } from "react";
|
|
2419
|
+
import { io as io2 } from "socket.io-client";
|
|
2419
2420
|
|
|
2420
2421
|
// src/utils/retryLookup.ts
|
|
2421
2422
|
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -2989,629 +2990,622 @@ function useTourPlayback({
|
|
|
2989
2990
|
useEffect11(() => {
|
|
2990
2991
|
if (disabled) return;
|
|
2991
2992
|
if (typeof window === "undefined") return;
|
|
2992
|
-
let cancelled = false;
|
|
2993
2993
|
let createdSocket = null;
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
2994
|
+
const socket = io2(serverUrl, {
|
|
2995
|
+
path: "/socket.io",
|
|
2996
|
+
transports: resolveSocketIoTransports(serverUrl, "polling-first"),
|
|
2997
|
+
autoConnect: true,
|
|
2998
|
+
reconnection: true,
|
|
2999
|
+
reconnectionAttempts: 10,
|
|
3000
|
+
reconnectionDelay: 1e3
|
|
3001
|
+
});
|
|
3002
|
+
createdSocket = socket;
|
|
3003
|
+
socketRef.current = socket;
|
|
3004
|
+
socket.on("connect", () => {
|
|
3005
|
+
console.log("[TourClient] Connected to tour agent server:", socket.id);
|
|
3006
|
+
const profile = userProfileRef.current;
|
|
3007
|
+
if (websiteId && profile?.userId && socket.connected) {
|
|
3008
|
+
socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3005
3009
|
}
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3010
|
+
});
|
|
3011
|
+
socket.on("tour:server_state", (payload) => {
|
|
3012
|
+
if (typeof payload?.runId === "number") {
|
|
3013
|
+
runIdRef.current = payload.runId;
|
|
3014
|
+
}
|
|
3015
|
+
if (typeof payload?.turnId === "string" || payload?.turnId === null) {
|
|
3016
|
+
turnIdRef.current = payload.turnId ?? null;
|
|
3017
|
+
}
|
|
3018
|
+
setServerState(payload);
|
|
3019
|
+
});
|
|
3020
|
+
socket.on("tour:command_cancel", (payload) => {
|
|
3021
|
+
console.log("[TourClient] Received command_cancel:", payload);
|
|
3022
|
+
if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
|
|
3023
|
+
activeCommandBatchIdRef.current = null;
|
|
3024
|
+
activeExecutionTokenRef.current++;
|
|
3025
|
+
commandInFlightRef.current = false;
|
|
3026
|
+
setPlaybackState("idle");
|
|
3027
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3028
|
+
window.speechSynthesis.cancel();
|
|
3021
3029
|
}
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
if (
|
|
3030
|
+
}
|
|
3031
|
+
});
|
|
3032
|
+
socket.on("tour:command", async (payload) => {
|
|
3033
|
+
const emitIfOpen = (ev, data) => {
|
|
3034
|
+
if (socketRef.current !== socket) return;
|
|
3035
|
+
if (!socket.connected) return;
|
|
3036
|
+
socket.emit(ev, data);
|
|
3037
|
+
};
|
|
3038
|
+
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3039
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3040
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3041
|
+
if (voiceInputResolveRef.current) {
|
|
3042
|
+
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3043
|
+
voiceInputResolveRef.current = null;
|
|
3044
|
+
resolvePendingVoiceInput("");
|
|
3045
|
+
}
|
|
3046
|
+
setPlaybackState("executing");
|
|
3047
|
+
commandInFlightRef.current = true;
|
|
3048
|
+
const commandBatchId = payload.commandBatchId ?? null;
|
|
3049
|
+
turnIdRef.current = payload.turnId ?? turnIdRef.current;
|
|
3050
|
+
const clearCommandBatchId = () => {
|
|
3051
|
+
if (activeCommandBatchIdRef.current === commandBatchId) {
|
|
3027
3052
|
activeCommandBatchIdRef.current = null;
|
|
3028
|
-
activeExecutionTokenRef.current++;
|
|
3029
|
-
commandInFlightRef.current = false;
|
|
3030
|
-
setPlaybackState("idle");
|
|
3031
|
-
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3032
|
-
window.speechSynthesis.cancel();
|
|
3033
|
-
}
|
|
3034
3053
|
}
|
|
3035
|
-
}
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
}
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
const
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3054
|
+
};
|
|
3055
|
+
activeCommandBatchIdRef.current = commandBatchId;
|
|
3056
|
+
const executionToken = ++activeExecutionTokenRef.current;
|
|
3057
|
+
const activeTourId = tourRef.current?.id;
|
|
3058
|
+
const activePreviewRunId = previewRunIdRef.current;
|
|
3059
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3060
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3061
|
+
stepOrder: payload.stepIndex,
|
|
3062
|
+
eventType: "command_batch_received",
|
|
3063
|
+
payload: {
|
|
3064
|
+
commandBatchId,
|
|
3065
|
+
commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
|
|
3066
|
+
},
|
|
3067
|
+
currentStepOrder: payload.stepIndex
|
|
3068
|
+
});
|
|
3069
|
+
}
|
|
3070
|
+
if (typeof payload.stepIndex === "number") {
|
|
3071
|
+
const prevStep = stepIndexRef.current;
|
|
3072
|
+
stepIndexRef.current = payload.stepIndex;
|
|
3073
|
+
setCurrentStepIndex(payload.stepIndex);
|
|
3074
|
+
if (payload.stepIndex !== prevStep) {
|
|
3075
|
+
const tour = tourRef.current;
|
|
3076
|
+
const total = tour?.steps?.length ?? 0;
|
|
3077
|
+
if (tour && total > 0) {
|
|
3078
|
+
onStepChangeRef.current?.(payload.stepIndex, total, tour);
|
|
3057
3079
|
}
|
|
3058
|
-
}
|
|
3059
|
-
|
|
3060
|
-
const executionToken = ++activeExecutionTokenRef.current;
|
|
3061
|
-
const activeTourId = tourRef.current?.id;
|
|
3062
|
-
const activePreviewRunId = previewRunIdRef.current;
|
|
3063
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3080
|
+
}
|
|
3081
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3064
3082
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3065
3083
|
stepOrder: payload.stepIndex,
|
|
3066
|
-
eventType: "
|
|
3084
|
+
eventType: "step_started",
|
|
3067
3085
|
payload: {
|
|
3068
|
-
|
|
3069
|
-
|
|
3086
|
+
previousStepOrder: prevStep,
|
|
3087
|
+
stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
|
|
3070
3088
|
},
|
|
3071
3089
|
currentStepOrder: payload.stepIndex
|
|
3072
3090
|
});
|
|
3073
3091
|
}
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
}
|
|
3097
|
-
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3098
|
-
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3099
|
-
commandInFlightRef.current = false;
|
|
3100
|
-
emitIfOpen("tour:action_result", {
|
|
3101
|
-
success: false,
|
|
3102
|
-
reason: "invalid_commands",
|
|
3103
|
-
commandBatchId,
|
|
3104
|
-
runId: runIdRef.current,
|
|
3105
|
-
turnId: turnIdRef.current
|
|
3106
|
-
});
|
|
3107
|
-
clearCommandBatchId();
|
|
3108
|
-
return;
|
|
3092
|
+
}
|
|
3093
|
+
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3094
|
+
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3095
|
+
commandInFlightRef.current = false;
|
|
3096
|
+
emitIfOpen("tour:action_result", {
|
|
3097
|
+
success: false,
|
|
3098
|
+
reason: "invalid_commands",
|
|
3099
|
+
commandBatchId,
|
|
3100
|
+
runId: runIdRef.current,
|
|
3101
|
+
turnId: turnIdRef.current
|
|
3102
|
+
});
|
|
3103
|
+
clearCommandBatchId();
|
|
3104
|
+
return;
|
|
3105
|
+
}
|
|
3106
|
+
let shouldWait = false;
|
|
3107
|
+
const results = [];
|
|
3108
|
+
let batchPreferredWaitTarget = null;
|
|
3109
|
+
const assertNotInterrupted = () => {
|
|
3110
|
+
if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
|
|
3111
|
+
const error = new Error("interrupted");
|
|
3112
|
+
error.code = "INTERRUPTED";
|
|
3113
|
+
throw error;
|
|
3109
3114
|
}
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
}
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3125
|
-
onRetry: () => {
|
|
3126
|
-
assertNotInterrupted();
|
|
3127
|
-
},
|
|
3128
|
-
resolve: async () => {
|
|
3129
|
-
let targetEl = null;
|
|
3130
|
-
if (params.uid) {
|
|
3131
|
-
const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
|
|
3132
|
-
targetEl = getElementByUid(params.uid);
|
|
3133
|
-
}
|
|
3134
|
-
if (!targetEl) {
|
|
3135
|
-
targetEl = resolveElementFromHints({
|
|
3136
|
-
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3137
|
-
testId: params.testId ?? fallbackHints?.testId,
|
|
3138
|
-
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3139
|
-
}, fallbackStep, tagStore);
|
|
3140
|
-
}
|
|
3141
|
-
return targetEl;
|
|
3142
|
-
}
|
|
3143
|
-
});
|
|
3144
|
-
};
|
|
3145
|
-
const executeOne = async (action) => {
|
|
3146
|
-
assertNotInterrupted();
|
|
3147
|
-
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3148
|
-
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3149
|
-
const executeTimeline = async (params = {}) => {
|
|
3150
|
-
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3151
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
3152
|
-
assertNotInterrupted();
|
|
3153
|
-
const segment = segments[index];
|
|
3154
|
-
const segmentText = (segment?.text ?? "").trim();
|
|
3155
|
-
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3156
|
-
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3157
|
-
if (segmentDelayMs > 0) {
|
|
3158
|
-
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3159
|
-
}
|
|
3160
|
-
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3161
|
-
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3162
|
-
if (!gateTarget) {
|
|
3163
|
-
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3164
|
-
}
|
|
3165
|
-
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3166
|
-
}
|
|
3167
|
-
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3168
|
-
showCaption(segmentText);
|
|
3169
|
-
}
|
|
3170
|
-
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3171
|
-
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3172
|
-
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3173
|
-
onNearEnd: nextSegmentText ? () => {
|
|
3174
|
-
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3175
|
-
} : void 0
|
|
3176
|
-
}) : Promise.resolve();
|
|
3177
|
-
const eventsPromise = (async () => {
|
|
3178
|
-
for (const event of events) {
|
|
3179
|
-
assertNotInterrupted();
|
|
3180
|
-
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3181
|
-
if (delayMs > 0) {
|
|
3182
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3183
|
-
}
|
|
3184
|
-
if (event?.action) {
|
|
3185
|
-
await executeOne(event.action);
|
|
3186
|
-
}
|
|
3187
|
-
}
|
|
3188
|
-
})();
|
|
3189
|
-
await Promise.all([speechPromise, eventsPromise]);
|
|
3190
|
-
}
|
|
3191
|
-
if (params.removeHighlightAtEnd !== false) {
|
|
3192
|
-
removeHighlight();
|
|
3193
|
-
}
|
|
3194
|
-
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3195
|
-
removeCaption();
|
|
3196
|
-
}
|
|
3197
|
-
return !!params.waitForInput;
|
|
3198
|
-
};
|
|
3199
|
-
if (action.type === "speak") {
|
|
3200
|
-
const text = action.params?.text ?? "";
|
|
3201
|
-
if (!text.trim()) {
|
|
3202
|
-
return { result: null };
|
|
3115
|
+
};
|
|
3116
|
+
const resolveTargetElement2 = async (params = {}, fallbackStep) => {
|
|
3117
|
+
const fallbackHints = fallbackStep?.element ?? null;
|
|
3118
|
+
return retryLookup({
|
|
3119
|
+
timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
|
|
3120
|
+
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3121
|
+
onRetry: () => {
|
|
3122
|
+
assertNotInterrupted();
|
|
3123
|
+
},
|
|
3124
|
+
resolve: async () => {
|
|
3125
|
+
let targetEl = null;
|
|
3126
|
+
if (params.uid) {
|
|
3127
|
+
const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
|
|
3128
|
+
targetEl = getElementByUid(params.uid);
|
|
3203
3129
|
}
|
|
3204
|
-
if (
|
|
3205
|
-
|
|
3130
|
+
if (!targetEl) {
|
|
3131
|
+
targetEl = resolveElementFromHints({
|
|
3132
|
+
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3133
|
+
testId: params.testId ?? fallbackHints?.testId,
|
|
3134
|
+
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3135
|
+
}, fallbackStep, tagStore);
|
|
3206
3136
|
}
|
|
3207
|
-
|
|
3208
|
-
// Schedule narration immediately so the client keeps its speculative,
|
|
3209
|
-
// low-latency behavior, but defer batch completion until playback settles.
|
|
3210
|
-
waitForCompletion: false,
|
|
3211
|
-
interrupt: action.params?.interrupt
|
|
3212
|
-
});
|
|
3213
|
-
return { result: text, settlePromise };
|
|
3214
|
-
}
|
|
3215
|
-
if (action.type === "play_timeline") {
|
|
3216
|
-
const timelineShouldWait = await executeTimeline(action.params);
|
|
3217
|
-
if (timelineShouldWait) shouldWait = true;
|
|
3218
|
-
return { result: "timeline_executed" };
|
|
3137
|
+
return targetEl;
|
|
3219
3138
|
}
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3139
|
+
});
|
|
3140
|
+
};
|
|
3141
|
+
const executeOne = async (action) => {
|
|
3142
|
+
assertNotInterrupted();
|
|
3143
|
+
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3144
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3145
|
+
const executeTimeline = async (params = {}) => {
|
|
3146
|
+
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3147
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
3148
|
+
assertNotInterrupted();
|
|
3149
|
+
const segment = segments[index];
|
|
3150
|
+
const segmentText = (segment?.text ?? "").trim();
|
|
3151
|
+
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3152
|
+
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3153
|
+
if (segmentDelayMs > 0) {
|
|
3154
|
+
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3155
|
+
}
|
|
3156
|
+
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3157
|
+
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3158
|
+
if (!gateTarget) {
|
|
3159
|
+
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3225
3160
|
}
|
|
3226
|
-
|
|
3227
|
-
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3228
|
-
return { result: "highlighted" };
|
|
3161
|
+
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3229
3162
|
}
|
|
3230
|
-
if (
|
|
3231
|
-
|
|
3232
|
-
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3233
|
-
);
|
|
3163
|
+
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3164
|
+
showCaption(segmentText);
|
|
3234
3165
|
}
|
|
3235
|
-
|
|
3166
|
+
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3167
|
+
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3168
|
+
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3169
|
+
onNearEnd: nextSegmentText ? () => {
|
|
3170
|
+
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3171
|
+
} : void 0
|
|
3172
|
+
}) : Promise.resolve();
|
|
3173
|
+
const eventsPromise = (async () => {
|
|
3174
|
+
for (const event of events) {
|
|
3175
|
+
assertNotInterrupted();
|
|
3176
|
+
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3177
|
+
if (delayMs > 0) {
|
|
3178
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3179
|
+
}
|
|
3180
|
+
if (event?.action) {
|
|
3181
|
+
await executeOne(event.action);
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
})();
|
|
3185
|
+
await Promise.all([speechPromise, eventsPromise]);
|
|
3236
3186
|
}
|
|
3237
|
-
if (
|
|
3187
|
+
if (params.removeHighlightAtEnd !== false) {
|
|
3238
3188
|
removeHighlight();
|
|
3239
|
-
return { result: "highlight_removed" };
|
|
3240
3189
|
}
|
|
3241
|
-
if (
|
|
3242
|
-
|
|
3243
|
-
if (!targetEl) {
|
|
3244
|
-
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3245
|
-
throw new Error(
|
|
3246
|
-
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3247
|
-
);
|
|
3248
|
-
}
|
|
3249
|
-
removeHighlight();
|
|
3250
|
-
await performInteractiveClick(targetEl);
|
|
3251
|
-
const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3252
|
-
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3253
|
-
return { result: "clicked" };
|
|
3190
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3191
|
+
removeCaption();
|
|
3254
3192
|
}
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
}
|
|
3262
|
-
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3263
|
-
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3264
|
-
await performInteractiveFill(targetEl, value);
|
|
3265
|
-
return { result: value };
|
|
3193
|
+
return !!params.waitForInput;
|
|
3194
|
+
};
|
|
3195
|
+
if (action.type === "speak") {
|
|
3196
|
+
const text = action.params?.text ?? "";
|
|
3197
|
+
if (!text.trim()) {
|
|
3198
|
+
return { result: null };
|
|
3266
3199
|
}
|
|
3267
|
-
if (
|
|
3268
|
-
|
|
3269
|
-
const html2canvas2 = html2canvasModule.default;
|
|
3270
|
-
const canvas = await html2canvas2(document.body, {
|
|
3271
|
-
useCORS: true,
|
|
3272
|
-
allowTaint: true,
|
|
3273
|
-
scale: Math.min(window.devicePixelRatio, 2),
|
|
3274
|
-
width: window.innerWidth,
|
|
3275
|
-
height: window.innerHeight,
|
|
3276
|
-
x: window.scrollX,
|
|
3277
|
-
y: window.scrollY,
|
|
3278
|
-
logging: false
|
|
3279
|
-
});
|
|
3280
|
-
return { result: canvas.toDataURL("image/png") };
|
|
3200
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3201
|
+
showCaption(text);
|
|
3281
3202
|
}
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3203
|
+
const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
|
|
3204
|
+
// Schedule narration immediately so the client keeps its speculative,
|
|
3205
|
+
// low-latency behavior, but defer batch completion until playback settles.
|
|
3206
|
+
waitForCompletion: false,
|
|
3207
|
+
interrupt: action.params?.interrupt
|
|
3208
|
+
});
|
|
3209
|
+
return { result: text, settlePromise };
|
|
3210
|
+
}
|
|
3211
|
+
if (action.type === "play_timeline") {
|
|
3212
|
+
const timelineShouldWait = await executeTimeline(action.params);
|
|
3213
|
+
if (timelineShouldWait) shouldWait = true;
|
|
3214
|
+
return { result: "timeline_executed" };
|
|
3215
|
+
}
|
|
3216
|
+
if (action.type === "highlight_element") {
|
|
3217
|
+
const resolvedEl = await resolveTargetElement2(action.params, currentStep);
|
|
3218
|
+
if (resolvedEl) {
|
|
3219
|
+
if (isEditableWaitTarget(resolvedEl)) {
|
|
3220
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
|
|
3286
3221
|
}
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
return { result: nextUrl };
|
|
3222
|
+
showHighlight(resolvedEl, action.params?.label || action.label);
|
|
3223
|
+
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3224
|
+
return { result: "highlighted" };
|
|
3291
3225
|
}
|
|
3292
|
-
if (action.
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
}
|
|
3297
|
-
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3298
|
-
const res = await fetch(url, {
|
|
3299
|
-
method: "POST",
|
|
3300
|
-
headers: { "Content-Type": "application/json" },
|
|
3301
|
-
body: JSON.stringify({
|
|
3302
|
-
command: action.params?.command,
|
|
3303
|
-
socketId: agentSocketId
|
|
3304
|
-
})
|
|
3305
|
-
});
|
|
3306
|
-
if (!res.ok) {
|
|
3307
|
-
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3308
|
-
}
|
|
3309
|
-
if (action.params?.wait !== false) {
|
|
3310
|
-
await res.json();
|
|
3311
|
-
}
|
|
3312
|
-
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3313
|
-
await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3314
|
-
await syncAOM();
|
|
3315
|
-
return { result: action.params?.command ?? "executed" };
|
|
3226
|
+
if (!action.params?.optional) {
|
|
3227
|
+
throw new Error(
|
|
3228
|
+
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3229
|
+
);
|
|
3316
3230
|
}
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3231
|
+
return { result: "highlight_optional_miss" };
|
|
3232
|
+
}
|
|
3233
|
+
if (action.type === "remove_highlight") {
|
|
3234
|
+
removeHighlight();
|
|
3235
|
+
return { result: "highlight_removed" };
|
|
3236
|
+
}
|
|
3237
|
+
if (action.type === "click_element") {
|
|
3238
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3239
|
+
if (!targetEl) {
|
|
3240
|
+
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3241
|
+
throw new Error(
|
|
3242
|
+
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3243
|
+
);
|
|
3325
3244
|
}
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3245
|
+
removeHighlight();
|
|
3246
|
+
await performInteractiveClick(targetEl);
|
|
3247
|
+
const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3248
|
+
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3249
|
+
return { result: "clicked" };
|
|
3250
|
+
}
|
|
3251
|
+
if (action.type === "fill_input") {
|
|
3252
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3253
|
+
if (!targetEl) {
|
|
3254
|
+
throw new Error(
|
|
3255
|
+
`fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3256
|
+
);
|
|
3329
3257
|
}
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3258
|
+
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3259
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3260
|
+
await performInteractiveFill(targetEl, value);
|
|
3261
|
+
return { result: value };
|
|
3262
|
+
}
|
|
3263
|
+
if (action.type === "take_screenshot") {
|
|
3264
|
+
const html2canvasModule = await import("html2canvas");
|
|
3265
|
+
const html2canvas2 = html2canvasModule.default;
|
|
3266
|
+
const canvas = await html2canvas2(document.body, {
|
|
3267
|
+
useCORS: true,
|
|
3268
|
+
allowTaint: true,
|
|
3269
|
+
scale: Math.min(window.devicePixelRatio, 2),
|
|
3270
|
+
width: window.innerWidth,
|
|
3271
|
+
height: window.innerHeight,
|
|
3272
|
+
x: window.scrollX,
|
|
3273
|
+
y: window.scrollY,
|
|
3274
|
+
logging: false
|
|
3275
|
+
});
|
|
3276
|
+
return { result: canvas.toDataURL("image/png") };
|
|
3277
|
+
}
|
|
3278
|
+
if (action.type === "navigate_to_url") {
|
|
3279
|
+
const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
|
|
3280
|
+
if (!nextUrl) {
|
|
3281
|
+
throw new Error("navigate_to_url missing url");
|
|
3333
3282
|
}
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
const isTerminal = isTerminalAction(command);
|
|
3344
|
-
if (isTerminal) {
|
|
3345
|
-
await Promise.all(pendingUIActions);
|
|
3346
|
-
pendingUIActions.length = 0;
|
|
3347
|
-
}
|
|
3348
|
-
const executionPromise = (async () => {
|
|
3349
|
-
const execution = await executeOne(command);
|
|
3350
|
-
await execution.settlePromise;
|
|
3351
|
-
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3352
|
-
})();
|
|
3353
|
-
if (isTerminal) {
|
|
3354
|
-
await executionPromise;
|
|
3355
|
-
} else {
|
|
3356
|
-
pendingUIActions.push(executionPromise);
|
|
3357
|
-
}
|
|
3283
|
+
await navigateToTourUrl(nextUrl);
|
|
3284
|
+
const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3285
|
+
await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3286
|
+
return { result: nextUrl };
|
|
3287
|
+
}
|
|
3288
|
+
if (action.type === "execute_agent_action") {
|
|
3289
|
+
const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
|
|
3290
|
+
if (!agentSocketId) {
|
|
3291
|
+
throw new Error("No socketId available for execute_agent_action");
|
|
3358
3292
|
}
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3293
|
+
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3294
|
+
const res = await fetch(url, {
|
|
3295
|
+
method: "POST",
|
|
3296
|
+
headers: { "Content-Type": "application/json" },
|
|
3297
|
+
body: JSON.stringify({
|
|
3298
|
+
command: action.params?.command,
|
|
3299
|
+
socketId: agentSocketId
|
|
3300
|
+
})
|
|
3362
3301
|
});
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
commandInFlightRef.current = false;
|
|
3366
|
-
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3367
|
-
if (interrupted) {
|
|
3368
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3369
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3370
|
-
stepOrder: stepIndexRef.current,
|
|
3371
|
-
eventType: "command_batch_interrupted",
|
|
3372
|
-
payload: {
|
|
3373
|
-
commandBatchId,
|
|
3374
|
-
partialResults: results
|
|
3375
|
-
},
|
|
3376
|
-
currentStepOrder: stepIndexRef.current
|
|
3377
|
-
});
|
|
3378
|
-
}
|
|
3379
|
-
emitIfOpen("tour:action_result", {
|
|
3380
|
-
success: true,
|
|
3381
|
-
interrupted: true,
|
|
3382
|
-
results,
|
|
3383
|
-
commandBatchId,
|
|
3384
|
-
runId: runIdRef.current,
|
|
3385
|
-
turnId: turnIdRef.current
|
|
3386
|
-
});
|
|
3387
|
-
clearCommandBatchId();
|
|
3388
|
-
return;
|
|
3302
|
+
if (!res.ok) {
|
|
3303
|
+
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3389
3304
|
}
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3393
|
-
stepOrder: stepIndexRef.current,
|
|
3394
|
-
eventType: "command_batch_failed",
|
|
3395
|
-
payload: {
|
|
3396
|
-
commandBatchId,
|
|
3397
|
-
error: String(err),
|
|
3398
|
-
partialResults: results
|
|
3399
|
-
},
|
|
3400
|
-
status: "active",
|
|
3401
|
-
currentStepOrder: stepIndexRef.current
|
|
3402
|
-
});
|
|
3305
|
+
if (action.params?.wait !== false) {
|
|
3306
|
+
await res.json();
|
|
3403
3307
|
}
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
results,
|
|
3409
|
-
commandBatchId,
|
|
3410
|
-
runId: runIdRef.current,
|
|
3411
|
-
turnId: turnIdRef.current
|
|
3412
|
-
});
|
|
3413
|
-
clearCommandBatchId();
|
|
3414
|
-
return;
|
|
3308
|
+
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3309
|
+
await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3310
|
+
await syncAOM();
|
|
3311
|
+
return { result: action.params?.command ?? "executed" };
|
|
3415
3312
|
}
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3421
|
-
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3422
|
-
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3423
|
-
let manualWaitPromise = null;
|
|
3424
|
-
let manualWaitKind = null;
|
|
3425
|
-
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3426
|
-
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3427
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3428
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3429
|
-
if (waitTargetHints) {
|
|
3430
|
-
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3431
|
-
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3432
|
-
manualWaitTarget = preferredWaitTarget;
|
|
3433
|
-
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3434
|
-
}
|
|
3435
|
-
if (manualWaitTarget) {
|
|
3436
|
-
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3437
|
-
manualWaitPromise = manualWait.promise;
|
|
3438
|
-
manualWaitKind = manualWait.kind;
|
|
3439
|
-
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3440
|
-
}
|
|
3313
|
+
if (action.type === "wait_for_user_action") {
|
|
3314
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3315
|
+
if (!targetEl) {
|
|
3316
|
+
throw new Error("wait_for_user_action target not found");
|
|
3441
3317
|
}
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3318
|
+
const eventName = action.params?.event ?? "click";
|
|
3319
|
+
await waitForUserAction(targetEl, eventName);
|
|
3320
|
+
return { result: `waited_for_${eventName}` };
|
|
3321
|
+
}
|
|
3322
|
+
if (action.type === "wait_for_input") {
|
|
3323
|
+
shouldWait = true;
|
|
3324
|
+
return { result: "waiting_for_input" };
|
|
3325
|
+
}
|
|
3326
|
+
if (action.type === "end_tour") {
|
|
3327
|
+
handleTourEnd();
|
|
3328
|
+
return { result: "ended" };
|
|
3329
|
+
}
|
|
3330
|
+
console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
|
|
3331
|
+
return { result: "unknown_action_skipped" };
|
|
3332
|
+
};
|
|
3333
|
+
try {
|
|
3334
|
+
const resultsBuffer = new Array(payload.commands.length);
|
|
3335
|
+
const pendingUIActions = [];
|
|
3336
|
+
for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
|
|
3337
|
+
const command = payload.commands[commandIndex];
|
|
3338
|
+
assertNotInterrupted();
|
|
3339
|
+
const isTerminal = isTerminalAction(command);
|
|
3340
|
+
if (isTerminal) {
|
|
3341
|
+
await Promise.all(pendingUIActions);
|
|
3342
|
+
pendingUIActions.length = 0;
|
|
3448
3343
|
}
|
|
3449
|
-
|
|
3450
|
-
const
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3459
|
-
}
|
|
3344
|
+
const executionPromise = (async () => {
|
|
3345
|
+
const execution = await executeOne(command);
|
|
3346
|
+
await execution.settlePromise;
|
|
3347
|
+
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3348
|
+
})();
|
|
3349
|
+
if (isTerminal) {
|
|
3350
|
+
await executionPromise;
|
|
3351
|
+
} else {
|
|
3352
|
+
pendingUIActions.push(executionPromise);
|
|
3460
3353
|
}
|
|
3461
|
-
|
|
3354
|
+
}
|
|
3355
|
+
await Promise.all(pendingUIActions);
|
|
3356
|
+
resultsBuffer.forEach((res) => {
|
|
3357
|
+
if (res) results.push(res);
|
|
3358
|
+
});
|
|
3359
|
+
await syncAOM();
|
|
3360
|
+
} catch (err) {
|
|
3361
|
+
commandInFlightRef.current = false;
|
|
3362
|
+
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3363
|
+
if (interrupted) {
|
|
3462
3364
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3463
3365
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3464
3366
|
stepOrder: stepIndexRef.current,
|
|
3465
|
-
eventType: "
|
|
3367
|
+
eventType: "command_batch_interrupted",
|
|
3466
3368
|
payload: {
|
|
3467
3369
|
commandBatchId,
|
|
3468
|
-
results
|
|
3370
|
+
partialResults: results
|
|
3469
3371
|
},
|
|
3470
3372
|
currentStepOrder: stepIndexRef.current
|
|
3471
3373
|
});
|
|
3472
3374
|
}
|
|
3473
3375
|
emitIfOpen("tour:action_result", {
|
|
3474
3376
|
success: true,
|
|
3475
|
-
|
|
3377
|
+
interrupted: true,
|
|
3476
3378
|
results,
|
|
3477
3379
|
commandBatchId,
|
|
3478
3380
|
runId: runIdRef.current,
|
|
3479
3381
|
turnId: turnIdRef.current
|
|
3480
3382
|
});
|
|
3481
3383
|
clearCommandBatchId();
|
|
3482
|
-
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3483
|
-
if (pendingInputBufRef.current) {
|
|
3484
|
-
const flushed = pendingInputBufRef.current;
|
|
3485
|
-
pendingInputBufRef.current = null;
|
|
3486
|
-
resolve(flushed);
|
|
3487
|
-
return;
|
|
3488
|
-
}
|
|
3489
|
-
voiceInputResolveRef.current = (text) => {
|
|
3490
|
-
voiceInputResolveRef.current = null;
|
|
3491
|
-
resolve(text);
|
|
3492
|
-
};
|
|
3493
|
-
});
|
|
3494
|
-
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3495
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3496
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3497
|
-
voiceInputResolveRef.current = null;
|
|
3498
|
-
setPlaybackState("executing");
|
|
3499
|
-
const transcript = userText.trim();
|
|
3500
|
-
if (!transcript) {
|
|
3501
|
-
return;
|
|
3502
|
-
}
|
|
3503
|
-
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3504
|
-
await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
|
|
3505
|
-
await syncAOM();
|
|
3506
|
-
emitIfOpen("tour:user_input", {
|
|
3507
|
-
transcript,
|
|
3508
|
-
runId: runIdRef.current,
|
|
3509
|
-
turnId: turnIdRef.current
|
|
3510
|
-
});
|
|
3511
|
-
});
|
|
3512
3384
|
return;
|
|
3513
3385
|
}
|
|
3386
|
+
console.error("[TourClient] Command batch execution failed:", err);
|
|
3514
3387
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3515
3388
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3516
3389
|
stepOrder: stepIndexRef.current,
|
|
3517
|
-
eventType: "
|
|
3390
|
+
eventType: "command_batch_failed",
|
|
3518
3391
|
payload: {
|
|
3519
3392
|
commandBatchId,
|
|
3520
|
-
|
|
3393
|
+
error: String(err),
|
|
3394
|
+
partialResults: results
|
|
3521
3395
|
},
|
|
3396
|
+
status: "active",
|
|
3522
3397
|
currentStepOrder: stepIndexRef.current
|
|
3523
3398
|
});
|
|
3524
3399
|
}
|
|
3525
3400
|
emitIfOpen("tour:action_result", {
|
|
3526
|
-
success:
|
|
3401
|
+
success: false,
|
|
3402
|
+
reason: "execution_error",
|
|
3403
|
+
error: String(err),
|
|
3527
3404
|
results,
|
|
3528
3405
|
commandBatchId,
|
|
3529
3406
|
runId: runIdRef.current,
|
|
3530
3407
|
turnId: turnIdRef.current
|
|
3531
3408
|
});
|
|
3532
3409
|
clearCommandBatchId();
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
const
|
|
3538
|
-
const
|
|
3539
|
-
|
|
3540
|
-
|
|
3541
|
-
|
|
3410
|
+
return;
|
|
3411
|
+
}
|
|
3412
|
+
commandInFlightRef.current = false;
|
|
3413
|
+
if (shouldWait && !skipRequestedRef.current) {
|
|
3414
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3415
|
+
const waitCondition = currentStep?.onboarding?.waitCondition;
|
|
3416
|
+
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3417
|
+
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3418
|
+
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3419
|
+
let manualWaitPromise = null;
|
|
3420
|
+
let manualWaitKind = null;
|
|
3421
|
+
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3422
|
+
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3423
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3424
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3425
|
+
if (waitTargetHints) {
|
|
3426
|
+
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3427
|
+
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3428
|
+
manualWaitTarget = preferredWaitTarget;
|
|
3429
|
+
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3430
|
+
}
|
|
3431
|
+
if (manualWaitTarget) {
|
|
3432
|
+
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3433
|
+
manualWaitPromise = manualWait.promise;
|
|
3434
|
+
manualWaitKind = manualWait.kind;
|
|
3435
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3436
|
+
}
|
|
3542
3437
|
}
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3438
|
+
if (!manualWaitPromise && preferredWaitTarget) {
|
|
3439
|
+
const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
|
|
3440
|
+
manualWaitPromise = manualWait.promise;
|
|
3441
|
+
manualWaitKind = manualWait.kind;
|
|
3442
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3443
|
+
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3444
|
+
}
|
|
3445
|
+
if (!manualWaitPromise && inputLikeWait) {
|
|
3446
|
+
const firstInput = document.querySelector(
|
|
3447
|
+
'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
|
|
3448
|
+
);
|
|
3449
|
+
if (firstInput) {
|
|
3450
|
+
const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
|
|
3451
|
+
manualWaitPromise = manualWait.promise;
|
|
3452
|
+
manualWaitKind = manualWait.kind;
|
|
3453
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3454
|
+
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3455
|
+
}
|
|
3456
|
+
}
|
|
3457
|
+
setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
|
|
3458
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3459
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3460
|
+
stepOrder: stepIndexRef.current,
|
|
3461
|
+
eventType: "waiting_for_input",
|
|
3557
3462
|
payload: {
|
|
3558
|
-
|
|
3559
|
-
|
|
3463
|
+
commandBatchId,
|
|
3464
|
+
results
|
|
3560
3465
|
},
|
|
3561
|
-
currentStepOrder:
|
|
3466
|
+
currentStepOrder: stepIndexRef.current
|
|
3562
3467
|
});
|
|
3563
3468
|
}
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3469
|
+
emitIfOpen("tour:action_result", {
|
|
3470
|
+
success: true,
|
|
3471
|
+
waitingForInput: true,
|
|
3472
|
+
results,
|
|
3473
|
+
commandBatchId,
|
|
3474
|
+
runId: runIdRef.current,
|
|
3475
|
+
turnId: turnIdRef.current
|
|
3476
|
+
});
|
|
3477
|
+
clearCommandBatchId();
|
|
3478
|
+
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3479
|
+
if (pendingInputBufRef.current) {
|
|
3480
|
+
const flushed = pendingInputBufRef.current;
|
|
3481
|
+
pendingInputBufRef.current = null;
|
|
3482
|
+
resolve(flushed);
|
|
3483
|
+
return;
|
|
3573
3484
|
}
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3485
|
+
voiceInputResolveRef.current = (text) => {
|
|
3486
|
+
voiceInputResolveRef.current = null;
|
|
3487
|
+
resolve(text);
|
|
3488
|
+
};
|
|
3489
|
+
});
|
|
3490
|
+
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3491
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3492
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3493
|
+
voiceInputResolveRef.current = null;
|
|
3494
|
+
setPlaybackState("executing");
|
|
3495
|
+
const transcript = userText.trim();
|
|
3496
|
+
if (!transcript) {
|
|
3497
|
+
return;
|
|
3498
|
+
}
|
|
3499
|
+
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3500
|
+
await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
|
|
3501
|
+
await syncAOM();
|
|
3502
|
+
emitIfOpen("tour:user_input", {
|
|
3503
|
+
transcript,
|
|
3504
|
+
runId: runIdRef.current,
|
|
3505
|
+
turnId: turnIdRef.current
|
|
3506
|
+
});
|
|
3507
|
+
});
|
|
3508
|
+
return;
|
|
3509
|
+
}
|
|
3510
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3511
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3512
|
+
stepOrder: stepIndexRef.current,
|
|
3513
|
+
eventType: "command_batch_completed",
|
|
3514
|
+
payload: {
|
|
3515
|
+
commandBatchId,
|
|
3516
|
+
results
|
|
3517
|
+
},
|
|
3518
|
+
currentStepOrder: stepIndexRef.current
|
|
3519
|
+
});
|
|
3520
|
+
}
|
|
3521
|
+
emitIfOpen("tour:action_result", {
|
|
3522
|
+
success: true,
|
|
3523
|
+
results,
|
|
3524
|
+
commandBatchId,
|
|
3525
|
+
runId: runIdRef.current,
|
|
3526
|
+
turnId: turnIdRef.current
|
|
3577
3527
|
});
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3528
|
+
clearCommandBatchId();
|
|
3529
|
+
});
|
|
3530
|
+
socket.on("tour:start", async (tourData) => {
|
|
3531
|
+
if (isActiveRef.current) return;
|
|
3532
|
+
runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
|
|
3533
|
+
const tour = tourData.tourContext ?? tourRef.current;
|
|
3534
|
+
const expType = experienceTypeRef.current;
|
|
3535
|
+
if (tour?.type && tour.type !== expType) {
|
|
3536
|
+
console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
|
|
3537
|
+
return;
|
|
3538
|
+
}
|
|
3539
|
+
skipRequestedRef.current = false;
|
|
3540
|
+
const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
|
|
3541
|
+
isActiveRef.current = true;
|
|
3542
|
+
setIsActive(true);
|
|
3543
|
+
setActiveTour(tour ?? null);
|
|
3544
|
+
tourRef.current = tour ?? null;
|
|
3545
|
+
setTotalSteps(total);
|
|
3546
|
+
stepIndexRef.current = 0;
|
|
3547
|
+
setCurrentStepIndex(0);
|
|
3548
|
+
setPlaybackState("intro");
|
|
3549
|
+
if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
|
|
3550
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
|
|
3551
|
+
stepOrder: 0,
|
|
3552
|
+
eventType: "tour_started",
|
|
3553
|
+
payload: {
|
|
3554
|
+
totalSteps: total,
|
|
3555
|
+
source: "sdk_test_preview"
|
|
3556
|
+
},
|
|
3557
|
+
currentStepOrder: 0
|
|
3558
|
+
});
|
|
3559
|
+
}
|
|
3560
|
+
try {
|
|
3561
|
+
const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
|
|
3562
|
+
const aom = generateMinifiedAOM2();
|
|
3563
|
+
if (socketRef.current === socket && socket.connected) {
|
|
3564
|
+
socket.emit("tour:sync_dom", {
|
|
3565
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3566
|
+
aom: aom.nodes,
|
|
3567
|
+
domSummary: captureDomSummary()
|
|
3568
|
+
});
|
|
3582
3569
|
}
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3570
|
+
} catch (e) {
|
|
3571
|
+
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3572
|
+
}
|
|
3573
|
+
});
|
|
3574
|
+
socket.on("tour:update", (payload) => {
|
|
3575
|
+
const updatedTour = payload?.tourContext;
|
|
3576
|
+
if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
|
|
3577
|
+
return;
|
|
3578
|
+
}
|
|
3579
|
+
tourRef.current = updatedTour;
|
|
3580
|
+
setActiveTour(updatedTour);
|
|
3581
|
+
const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
|
|
3582
|
+
setTotalSteps(nextTotal);
|
|
3583
|
+
if (typeof payload.currentStepIndex === "number") {
|
|
3584
|
+
const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
|
|
3585
|
+
stepIndexRef.current = clampedStepIndex;
|
|
3586
|
+
setCurrentStepIndex(clampedStepIndex);
|
|
3587
|
+
if (nextTotal > 0) {
|
|
3588
|
+
onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
|
|
3594
3589
|
}
|
|
3595
|
-
}
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
}
|
|
3608
|
-
}
|
|
3590
|
+
}
|
|
3591
|
+
});
|
|
3592
|
+
socket.on("tour:end", () => {
|
|
3593
|
+
setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
|
|
3594
|
+
handleTourEnd();
|
|
3595
|
+
});
|
|
3596
|
+
socket.on("tour:debug_log", (entry) => {
|
|
3597
|
+
const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
|
|
3598
|
+
if (isDev) {
|
|
3599
|
+
console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
|
|
3600
|
+
if (typeof window !== "undefined") {
|
|
3601
|
+
window.dispatchEvent(new CustomEvent("modelnex-debug", {
|
|
3602
|
+
detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
|
|
3603
|
+
}));
|
|
3609
3604
|
}
|
|
3610
|
-
}
|
|
3611
|
-
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3605
|
+
}
|
|
3612
3606
|
});
|
|
3607
|
+
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3613
3608
|
return () => {
|
|
3614
|
-
cancelled = true;
|
|
3615
3609
|
const toClose = createdSocket ?? socketRef.current;
|
|
3616
3610
|
if (toClose) {
|
|
3617
3611
|
toClose.disconnect();
|
|
@@ -3628,7 +3622,12 @@ function useTourPlayback({
|
|
|
3628
3622
|
const s = socketRef.current;
|
|
3629
3623
|
const profile = userProfile;
|
|
3630
3624
|
if (!s?.connected || !websiteId || !profile?.userId) return;
|
|
3631
|
-
|
|
3625
|
+
const timer = setTimeout(() => {
|
|
3626
|
+
if (s.connected) {
|
|
3627
|
+
s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3628
|
+
}
|
|
3629
|
+
}, 150);
|
|
3630
|
+
return () => clearTimeout(timer);
|
|
3632
3631
|
}, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
|
|
3633
3632
|
useEffect11(() => {
|
|
3634
3633
|
if (!showCaptions || !isReviewMode) {
|
|
@@ -3653,11 +3652,13 @@ function useTourPlayback({
|
|
|
3653
3652
|
if (!socketRef.current?.connected || !isActiveRef.current) return;
|
|
3654
3653
|
const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
|
|
3655
3654
|
const aom = generateMinifiedAOM2();
|
|
3656
|
-
socketRef.current
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3655
|
+
if (socketRef.current?.connected) {
|
|
3656
|
+
socketRef.current.emit("tour:sync_dom", {
|
|
3657
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3658
|
+
aom: aom.nodes,
|
|
3659
|
+
domSummary: captureDomSummary()
|
|
3660
|
+
});
|
|
3661
|
+
}
|
|
3661
3662
|
}, []);
|
|
3662
3663
|
const interruptExecution = useCallback7((transcript) => {
|
|
3663
3664
|
if (!socketRef.current?.connected || !isActiveRef.current) return false;
|
|
@@ -3668,21 +3669,23 @@ function useTourPlayback({
|
|
|
3668
3669
|
removeHighlight();
|
|
3669
3670
|
removeCaption();
|
|
3670
3671
|
voice.stopSpeaking();
|
|
3671
|
-
socketRef.current
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3672
|
+
if (socketRef.current?.connected) {
|
|
3673
|
+
socketRef.current.emit("tour:action_result", {
|
|
3674
|
+
success: true,
|
|
3675
|
+
interrupted: true,
|
|
3676
|
+
transcript,
|
|
3677
|
+
commandBatchId: activeCommandBatchIdRef.current,
|
|
3678
|
+
runId: runIdRef.current,
|
|
3679
|
+
turnId: turnIdRef.current
|
|
3680
|
+
});
|
|
3681
|
+
activeCommandBatchIdRef.current = null;
|
|
3682
|
+
socketRef.current.emit("tour:user_input", {
|
|
3683
|
+
transcript,
|
|
3684
|
+
interrupted: true,
|
|
3685
|
+
runId: runIdRef.current,
|
|
3686
|
+
turnId: turnIdRef.current
|
|
3687
|
+
});
|
|
3688
|
+
}
|
|
3686
3689
|
setPlaybackState("thinking");
|
|
3687
3690
|
return true;
|
|
3688
3691
|
}, [voice]);
|
|
@@ -3800,11 +3803,13 @@ function useTourPlayback({
|
|
|
3800
3803
|
previewRunIdRef.current = null;
|
|
3801
3804
|
}
|
|
3802
3805
|
tourRef.current = tour;
|
|
3803
|
-
socketRef.current
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
3807
|
-
|
|
3806
|
+
if (socketRef.current?.connected) {
|
|
3807
|
+
socketRef.current.emit("tour:request_start", {
|
|
3808
|
+
tourId: tour.id,
|
|
3809
|
+
previewRunId: previewRunIdRef.current,
|
|
3810
|
+
tourContext: tour
|
|
3811
|
+
});
|
|
3812
|
+
}
|
|
3808
3813
|
}, [serverUrl, websiteId]);
|
|
3809
3814
|
useEffect11(() => {
|
|
3810
3815
|
if (!enableAutoDiscovery) return;
|
|
@@ -5252,7 +5257,7 @@ function useVoice(serverUrl) {
|
|
|
5252
5257
|
(async () => {
|
|
5253
5258
|
try {
|
|
5254
5259
|
const ioModule = await import("socket.io-client");
|
|
5255
|
-
const
|
|
5260
|
+
const io3 = ioModule.default || ioModule;
|
|
5256
5261
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
5257
5262
|
audio: {
|
|
5258
5263
|
echoCancellation: true,
|
|
@@ -5274,7 +5279,7 @@ function useVoice(serverUrl) {
|
|
|
5274
5279
|
audioBitsPerSecond: 128e3
|
|
5275
5280
|
});
|
|
5276
5281
|
mediaRecorderRef.current = recorder;
|
|
5277
|
-
const socket =
|
|
5282
|
+
const socket = io3(serverUrl, {
|
|
5278
5283
|
path: "/socket.io",
|
|
5279
5284
|
transports: resolveSocketIoTransports(serverUrl, "websocket-first")
|
|
5280
5285
|
});
|
|
@@ -10794,7 +10799,7 @@ var ModelNexProvider = ({
|
|
|
10794
10799
|
socketId,
|
|
10795
10800
|
devMode
|
|
10796
10801
|
}),
|
|
10797
|
-
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
|
|
10802
|
+
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
|
|
10798
10803
|
);
|
|
10799
10804
|
return React8.createElement(
|
|
10800
10805
|
ModelNexContext.Provider,
|