@modelnex/sdk 0.5.14 → 0.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +589 -551
- package/dist/index.mjs +589 -551
- package/package.json +11 -12
- package/dist/aom-GA6W42DG.mjs +0 -71
- package/dist/aom-J6NYMGDW.mjs +0 -69
- package/dist/chunk-6DZX6EAA.mjs +0 -37
- package/dist/dom-sync-Y7Z7TOU6.mjs +0 -57
package/dist/index.mjs
CHANGED
|
@@ -2114,7 +2114,7 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
|
|
|
2114
2114
|
}
|
|
2115
2115
|
|
|
2116
2116
|
// src/constants.ts
|
|
2117
|
-
var DEFAULT_MODELNEX_SERVER_URL = "https://modelnex
|
|
2117
|
+
var DEFAULT_MODELNEX_SERVER_URL = "https://api.modelnex.com";
|
|
2118
2118
|
|
|
2119
2119
|
// src/hooks/useRunCommand.ts
|
|
2120
2120
|
import { useCallback as useCallback5, useContext as useContext2 } from "react";
|
|
@@ -2416,6 +2416,7 @@ function isTourEligible(tour, userProfile) {
|
|
|
2416
2416
|
|
|
2417
2417
|
// src/hooks/useTourPlayback.ts
|
|
2418
2418
|
import { useState as useState7, useRef as useRef8, useCallback as useCallback7, useEffect as useEffect11, useContext as useContext4 } from "react";
|
|
2419
|
+
import { io as io2 } from "socket.io-client";
|
|
2419
2420
|
|
|
2420
2421
|
// src/utils/retryLookup.ts
|
|
2421
2422
|
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -2944,6 +2945,12 @@ function useTourPlayback({
|
|
|
2944
2945
|
const [serverState, setServerState] = useState7(null);
|
|
2945
2946
|
const ctx = useContext4(ModelNexContext);
|
|
2946
2947
|
const devMode = ctx?.devMode;
|
|
2948
|
+
const devModeRef = useRef8(devMode);
|
|
2949
|
+
devModeRef.current = devMode;
|
|
2950
|
+
const userProfileRef = useRef8(userProfile);
|
|
2951
|
+
userProfileRef.current = userProfile;
|
|
2952
|
+
const experienceTypeRef = useRef8(experienceType);
|
|
2953
|
+
experienceTypeRef.current = experienceType;
|
|
2947
2954
|
const tourRef = useRef8(null);
|
|
2948
2955
|
const stepIndexRef = useRef8(0);
|
|
2949
2956
|
const skipRequestedRef = useRef8(false);
|
|
@@ -2983,408 +2990,391 @@ function useTourPlayback({
|
|
|
2983
2990
|
useEffect11(() => {
|
|
2984
2991
|
if (disabled) return;
|
|
2985
2992
|
if (typeof window === "undefined") return;
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
socket.
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
2993
|
+
let createdSocket = null;
|
|
2994
|
+
const socket = io2(serverUrl, {
|
|
2995
|
+
path: "/socket.io",
|
|
2996
|
+
transports: resolveSocketIoTransports(serverUrl, "polling-first"),
|
|
2997
|
+
autoConnect: true,
|
|
2998
|
+
reconnection: true,
|
|
2999
|
+
reconnectionAttempts: 10,
|
|
3000
|
+
reconnectionDelay: 1e3
|
|
3001
|
+
});
|
|
3002
|
+
createdSocket = socket;
|
|
3003
|
+
socketRef.current = socket;
|
|
3004
|
+
socket.on("connect", () => {
|
|
3005
|
+
console.log("[TourClient] Connected to tour agent server:", socket.id);
|
|
3006
|
+
const profile = userProfileRef.current;
|
|
3007
|
+
if (websiteId && profile?.userId && socket.connected) {
|
|
3008
|
+
socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3009
|
+
}
|
|
3010
|
+
});
|
|
3011
|
+
socket.on("tour:server_state", (payload) => {
|
|
3012
|
+
if (typeof payload?.runId === "number") {
|
|
3013
|
+
runIdRef.current = payload.runId;
|
|
3014
|
+
}
|
|
3015
|
+
if (typeof payload?.turnId === "string" || payload?.turnId === null) {
|
|
3016
|
+
turnIdRef.current = payload.turnId ?? null;
|
|
3017
|
+
}
|
|
3018
|
+
setServerState(payload);
|
|
3019
|
+
});
|
|
3020
|
+
socket.on("tour:command_cancel", (payload) => {
|
|
3021
|
+
console.log("[TourClient] Received command_cancel:", payload);
|
|
3022
|
+
if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
|
|
3023
|
+
activeCommandBatchIdRef.current = null;
|
|
3024
|
+
activeExecutionTokenRef.current++;
|
|
3025
|
+
commandInFlightRef.current = false;
|
|
3026
|
+
setPlaybackState("idle");
|
|
3027
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3028
|
+
window.speechSynthesis.cancel();
|
|
3006
3029
|
}
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
if (
|
|
3030
|
+
}
|
|
3031
|
+
});
|
|
3032
|
+
socket.on("tour:command", async (payload) => {
|
|
3033
|
+
const emitIfOpen = (ev, data) => {
|
|
3034
|
+
if (socketRef.current !== socket) return;
|
|
3035
|
+
if (!socket.connected) return;
|
|
3036
|
+
socket.emit(ev, data);
|
|
3037
|
+
};
|
|
3038
|
+
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3039
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3040
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3041
|
+
if (voiceInputResolveRef.current) {
|
|
3042
|
+
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3043
|
+
voiceInputResolveRef.current = null;
|
|
3044
|
+
resolvePendingVoiceInput("");
|
|
3045
|
+
}
|
|
3046
|
+
setPlaybackState("executing");
|
|
3047
|
+
commandInFlightRef.current = true;
|
|
3048
|
+
const commandBatchId = payload.commandBatchId ?? null;
|
|
3049
|
+
turnIdRef.current = payload.turnId ?? turnIdRef.current;
|
|
3050
|
+
const clearCommandBatchId = () => {
|
|
3051
|
+
if (activeCommandBatchIdRef.current === commandBatchId) {
|
|
3012
3052
|
activeCommandBatchIdRef.current = null;
|
|
3013
|
-
activeExecutionTokenRef.current++;
|
|
3014
|
-
commandInFlightRef.current = false;
|
|
3015
|
-
setPlaybackState("idle");
|
|
3016
|
-
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3017
|
-
window.speechSynthesis.cancel();
|
|
3018
|
-
}
|
|
3019
3053
|
}
|
|
3020
|
-
}
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3054
|
+
};
|
|
3055
|
+
activeCommandBatchIdRef.current = commandBatchId;
|
|
3056
|
+
const executionToken = ++activeExecutionTokenRef.current;
|
|
3057
|
+
const activeTourId = tourRef.current?.id;
|
|
3058
|
+
const activePreviewRunId = previewRunIdRef.current;
|
|
3059
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3060
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3061
|
+
stepOrder: payload.stepIndex,
|
|
3062
|
+
eventType: "command_batch_received",
|
|
3063
|
+
payload: {
|
|
3064
|
+
commandBatchId,
|
|
3065
|
+
commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
|
|
3066
|
+
},
|
|
3067
|
+
currentStepOrder: payload.stepIndex
|
|
3068
|
+
});
|
|
3069
|
+
}
|
|
3070
|
+
if (typeof payload.stepIndex === "number") {
|
|
3071
|
+
const prevStep = stepIndexRef.current;
|
|
3072
|
+
stepIndexRef.current = payload.stepIndex;
|
|
3073
|
+
setCurrentStepIndex(payload.stepIndex);
|
|
3074
|
+
if (payload.stepIndex !== prevStep) {
|
|
3075
|
+
const tour = tourRef.current;
|
|
3076
|
+
const total = tour?.steps?.length ?? 0;
|
|
3077
|
+
if (tour && total > 0) {
|
|
3078
|
+
onStepChangeRef.current?.(payload.stepIndex, total, tour);
|
|
3037
3079
|
}
|
|
3038
|
-
}
|
|
3039
|
-
|
|
3040
|
-
const executionToken = ++activeExecutionTokenRef.current;
|
|
3041
|
-
const activeTourId = tourRef.current?.id;
|
|
3042
|
-
const activePreviewRunId = previewRunIdRef.current;
|
|
3043
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3080
|
+
}
|
|
3081
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3044
3082
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3045
3083
|
stepOrder: payload.stepIndex,
|
|
3046
|
-
eventType: "
|
|
3084
|
+
eventType: "step_started",
|
|
3047
3085
|
payload: {
|
|
3048
|
-
|
|
3049
|
-
|
|
3086
|
+
previousStepOrder: prevStep,
|
|
3087
|
+
stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
|
|
3050
3088
|
},
|
|
3051
3089
|
currentStepOrder: payload.stepIndex
|
|
3052
3090
|
});
|
|
3053
3091
|
}
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
}
|
|
3077
|
-
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3078
|
-
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3079
|
-
commandInFlightRef.current = false;
|
|
3080
|
-
socket.emit("tour:action_result", {
|
|
3081
|
-
success: false,
|
|
3082
|
-
reason: "invalid_commands",
|
|
3083
|
-
commandBatchId,
|
|
3084
|
-
runId: runIdRef.current,
|
|
3085
|
-
turnId: turnIdRef.current
|
|
3086
|
-
});
|
|
3087
|
-
clearCommandBatchId();
|
|
3088
|
-
return;
|
|
3092
|
+
}
|
|
3093
|
+
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3094
|
+
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3095
|
+
commandInFlightRef.current = false;
|
|
3096
|
+
emitIfOpen("tour:action_result", {
|
|
3097
|
+
success: false,
|
|
3098
|
+
reason: "invalid_commands",
|
|
3099
|
+
commandBatchId,
|
|
3100
|
+
runId: runIdRef.current,
|
|
3101
|
+
turnId: turnIdRef.current
|
|
3102
|
+
});
|
|
3103
|
+
clearCommandBatchId();
|
|
3104
|
+
return;
|
|
3105
|
+
}
|
|
3106
|
+
let shouldWait = false;
|
|
3107
|
+
const results = [];
|
|
3108
|
+
let batchPreferredWaitTarget = null;
|
|
3109
|
+
const assertNotInterrupted = () => {
|
|
3110
|
+
if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
|
|
3111
|
+
const error = new Error("interrupted");
|
|
3112
|
+
error.code = "INTERRUPTED";
|
|
3113
|
+
throw error;
|
|
3089
3114
|
}
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
}
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3105
|
-
onRetry: () => {
|
|
3106
|
-
assertNotInterrupted();
|
|
3107
|
-
},
|
|
3108
|
-
resolve: async () => {
|
|
3109
|
-
let targetEl = null;
|
|
3110
|
-
if (params.uid) {
|
|
3111
|
-
const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
|
|
3112
|
-
targetEl = getElementByUid(params.uid);
|
|
3113
|
-
}
|
|
3114
|
-
if (!targetEl) {
|
|
3115
|
-
targetEl = resolveElementFromHints({
|
|
3116
|
-
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3117
|
-
testId: params.testId ?? fallbackHints?.testId,
|
|
3118
|
-
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3119
|
-
}, fallbackStep, tagStore);
|
|
3120
|
-
}
|
|
3121
|
-
return targetEl;
|
|
3122
|
-
}
|
|
3123
|
-
});
|
|
3124
|
-
};
|
|
3125
|
-
const executeOne = async (action) => {
|
|
3126
|
-
assertNotInterrupted();
|
|
3127
|
-
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3128
|
-
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3129
|
-
const executeTimeline = async (params = {}) => {
|
|
3130
|
-
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3131
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
3132
|
-
assertNotInterrupted();
|
|
3133
|
-
const segment = segments[index];
|
|
3134
|
-
const segmentText = (segment?.text ?? "").trim();
|
|
3135
|
-
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3136
|
-
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3137
|
-
if (segmentDelayMs > 0) {
|
|
3138
|
-
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3139
|
-
}
|
|
3140
|
-
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3141
|
-
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3142
|
-
if (!gateTarget) {
|
|
3143
|
-
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3144
|
-
}
|
|
3145
|
-
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3146
|
-
}
|
|
3147
|
-
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3148
|
-
showCaption(segmentText);
|
|
3149
|
-
}
|
|
3150
|
-
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3151
|
-
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3152
|
-
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3153
|
-
onNearEnd: nextSegmentText ? () => {
|
|
3154
|
-
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3155
|
-
} : void 0
|
|
3156
|
-
}) : Promise.resolve();
|
|
3157
|
-
const eventsPromise = (async () => {
|
|
3158
|
-
for (const event of events) {
|
|
3159
|
-
assertNotInterrupted();
|
|
3160
|
-
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3161
|
-
if (delayMs > 0) {
|
|
3162
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3163
|
-
}
|
|
3164
|
-
if (event?.action) {
|
|
3165
|
-
await executeOne(event.action);
|
|
3166
|
-
}
|
|
3167
|
-
}
|
|
3168
|
-
})();
|
|
3169
|
-
await Promise.all([speechPromise, eventsPromise]);
|
|
3170
|
-
}
|
|
3171
|
-
if (params.removeHighlightAtEnd !== false) {
|
|
3172
|
-
removeHighlight();
|
|
3173
|
-
}
|
|
3174
|
-
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3175
|
-
removeCaption();
|
|
3176
|
-
}
|
|
3177
|
-
return !!params.waitForInput;
|
|
3178
|
-
};
|
|
3179
|
-
if (action.type === "speak") {
|
|
3180
|
-
const text = action.params?.text ?? "";
|
|
3181
|
-
if (!text.trim()) {
|
|
3182
|
-
return { result: null };
|
|
3115
|
+
};
|
|
3116
|
+
const resolveTargetElement2 = async (params = {}, fallbackStep) => {
|
|
3117
|
+
const fallbackHints = fallbackStep?.element ?? null;
|
|
3118
|
+
return retryLookup({
|
|
3119
|
+
timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
|
|
3120
|
+
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3121
|
+
onRetry: () => {
|
|
3122
|
+
assertNotInterrupted();
|
|
3123
|
+
},
|
|
3124
|
+
resolve: async () => {
|
|
3125
|
+
let targetEl = null;
|
|
3126
|
+
if (params.uid) {
|
|
3127
|
+
const { getElementByUid } = await import("./aom-HDYNCIOY.mjs");
|
|
3128
|
+
targetEl = getElementByUid(params.uid);
|
|
3183
3129
|
}
|
|
3184
|
-
if (
|
|
3185
|
-
|
|
3130
|
+
if (!targetEl) {
|
|
3131
|
+
targetEl = resolveElementFromHints({
|
|
3132
|
+
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3133
|
+
testId: params.testId ?? fallbackHints?.testId,
|
|
3134
|
+
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3135
|
+
}, fallbackStep, tagStore);
|
|
3186
3136
|
}
|
|
3187
|
-
|
|
3188
|
-
// Schedule narration immediately so the client keeps its speculative,
|
|
3189
|
-
// low-latency behavior, but defer batch completion until playback settles.
|
|
3190
|
-
waitForCompletion: false,
|
|
3191
|
-
interrupt: action.params?.interrupt
|
|
3192
|
-
});
|
|
3193
|
-
return { result: text, settlePromise };
|
|
3137
|
+
return targetEl;
|
|
3194
3138
|
}
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3139
|
+
});
|
|
3140
|
+
};
|
|
3141
|
+
const executeOne = async (action) => {
|
|
3142
|
+
assertNotInterrupted();
|
|
3143
|
+
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3144
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3145
|
+
const executeTimeline = async (params = {}) => {
|
|
3146
|
+
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3147
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
3148
|
+
assertNotInterrupted();
|
|
3149
|
+
const segment = segments[index];
|
|
3150
|
+
const segmentText = (segment?.text ?? "").trim();
|
|
3151
|
+
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3152
|
+
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3153
|
+
if (segmentDelayMs > 0) {
|
|
3154
|
+
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3155
|
+
}
|
|
3156
|
+
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3157
|
+
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3158
|
+
if (!gateTarget) {
|
|
3159
|
+
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3205
3160
|
}
|
|
3206
|
-
|
|
3207
|
-
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3208
|
-
return { result: "highlighted" };
|
|
3161
|
+
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3209
3162
|
}
|
|
3210
|
-
if (
|
|
3211
|
-
|
|
3212
|
-
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3213
|
-
);
|
|
3163
|
+
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3164
|
+
showCaption(segmentText);
|
|
3214
3165
|
}
|
|
3215
|
-
|
|
3166
|
+
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3167
|
+
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3168
|
+
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3169
|
+
onNearEnd: nextSegmentText ? () => {
|
|
3170
|
+
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3171
|
+
} : void 0
|
|
3172
|
+
}) : Promise.resolve();
|
|
3173
|
+
const eventsPromise = (async () => {
|
|
3174
|
+
for (const event of events) {
|
|
3175
|
+
assertNotInterrupted();
|
|
3176
|
+
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3177
|
+
if (delayMs > 0) {
|
|
3178
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3179
|
+
}
|
|
3180
|
+
if (event?.action) {
|
|
3181
|
+
await executeOne(event.action);
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
})();
|
|
3185
|
+
await Promise.all([speechPromise, eventsPromise]);
|
|
3216
3186
|
}
|
|
3217
|
-
if (
|
|
3187
|
+
if (params.removeHighlightAtEnd !== false) {
|
|
3218
3188
|
removeHighlight();
|
|
3219
|
-
return { result: "highlight_removed" };
|
|
3220
3189
|
}
|
|
3221
|
-
if (
|
|
3222
|
-
|
|
3223
|
-
if (!targetEl) {
|
|
3224
|
-
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3225
|
-
throw new Error(
|
|
3226
|
-
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3227
|
-
);
|
|
3228
|
-
}
|
|
3229
|
-
removeHighlight();
|
|
3230
|
-
await performInteractiveClick(targetEl);
|
|
3231
|
-
const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3232
|
-
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3233
|
-
return { result: "clicked" };
|
|
3190
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3191
|
+
removeCaption();
|
|
3234
3192
|
}
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
}
|
|
3242
|
-
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3243
|
-
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3244
|
-
await performInteractiveFill(targetEl, value);
|
|
3245
|
-
return { result: value };
|
|
3193
|
+
return !!params.waitForInput;
|
|
3194
|
+
};
|
|
3195
|
+
if (action.type === "speak") {
|
|
3196
|
+
const text = action.params?.text ?? "";
|
|
3197
|
+
if (!text.trim()) {
|
|
3198
|
+
return { result: null };
|
|
3246
3199
|
}
|
|
3247
|
-
if (
|
|
3248
|
-
|
|
3249
|
-
const html2canvas2 = html2canvasModule.default;
|
|
3250
|
-
const canvas = await html2canvas2(document.body, {
|
|
3251
|
-
useCORS: true,
|
|
3252
|
-
allowTaint: true,
|
|
3253
|
-
scale: Math.min(window.devicePixelRatio, 2),
|
|
3254
|
-
width: window.innerWidth,
|
|
3255
|
-
height: window.innerHeight,
|
|
3256
|
-
x: window.scrollX,
|
|
3257
|
-
y: window.scrollY,
|
|
3258
|
-
logging: false
|
|
3259
|
-
});
|
|
3260
|
-
return { result: canvas.toDataURL("image/png") };
|
|
3200
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3201
|
+
showCaption(text);
|
|
3261
3202
|
}
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3203
|
+
const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
|
|
3204
|
+
// Schedule narration immediately so the client keeps its speculative,
|
|
3205
|
+
// low-latency behavior, but defer batch completion until playback settles.
|
|
3206
|
+
waitForCompletion: false,
|
|
3207
|
+
interrupt: action.params?.interrupt
|
|
3208
|
+
});
|
|
3209
|
+
return { result: text, settlePromise };
|
|
3210
|
+
}
|
|
3211
|
+
if (action.type === "play_timeline") {
|
|
3212
|
+
const timelineShouldWait = await executeTimeline(action.params);
|
|
3213
|
+
if (timelineShouldWait) shouldWait = true;
|
|
3214
|
+
return { result: "timeline_executed" };
|
|
3215
|
+
}
|
|
3216
|
+
if (action.type === "highlight_element") {
|
|
3217
|
+
const resolvedEl = await resolveTargetElement2(action.params, currentStep);
|
|
3218
|
+
if (resolvedEl) {
|
|
3219
|
+
if (isEditableWaitTarget(resolvedEl)) {
|
|
3220
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
|
|
3266
3221
|
}
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
return { result: nextUrl };
|
|
3222
|
+
showHighlight(resolvedEl, action.params?.label || action.label);
|
|
3223
|
+
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3224
|
+
return { result: "highlighted" };
|
|
3271
3225
|
}
|
|
3272
|
-
if (action.
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
}
|
|
3277
|
-
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3278
|
-
const res = await fetch(url, {
|
|
3279
|
-
method: "POST",
|
|
3280
|
-
headers: { "Content-Type": "application/json" },
|
|
3281
|
-
body: JSON.stringify({
|
|
3282
|
-
command: action.params?.command,
|
|
3283
|
-
socketId: agentSocketId
|
|
3284
|
-
})
|
|
3285
|
-
});
|
|
3286
|
-
if (!res.ok) {
|
|
3287
|
-
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3288
|
-
}
|
|
3289
|
-
if (action.params?.wait !== false) {
|
|
3290
|
-
await res.json();
|
|
3291
|
-
}
|
|
3292
|
-
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3293
|
-
await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3294
|
-
await syncAOM();
|
|
3295
|
-
return { result: action.params?.command ?? "executed" };
|
|
3226
|
+
if (!action.params?.optional) {
|
|
3227
|
+
throw new Error(
|
|
3228
|
+
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3229
|
+
);
|
|
3296
3230
|
}
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3231
|
+
return { result: "highlight_optional_miss" };
|
|
3232
|
+
}
|
|
3233
|
+
if (action.type === "remove_highlight") {
|
|
3234
|
+
removeHighlight();
|
|
3235
|
+
return { result: "highlight_removed" };
|
|
3236
|
+
}
|
|
3237
|
+
if (action.type === "click_element") {
|
|
3238
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3239
|
+
if (!targetEl) {
|
|
3240
|
+
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3241
|
+
throw new Error(
|
|
3242
|
+
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3243
|
+
);
|
|
3305
3244
|
}
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3245
|
+
removeHighlight();
|
|
3246
|
+
await performInteractiveClick(targetEl);
|
|
3247
|
+
const { waitForDomSettle: waitForDomSettleClick } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3248
|
+
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3249
|
+
return { result: "clicked" };
|
|
3250
|
+
}
|
|
3251
|
+
if (action.type === "fill_input") {
|
|
3252
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3253
|
+
if (!targetEl) {
|
|
3254
|
+
throw new Error(
|
|
3255
|
+
`fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3256
|
+
);
|
|
3309
3257
|
}
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3258
|
+
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3259
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3260
|
+
await performInteractiveFill(targetEl, value);
|
|
3261
|
+
return { result: value };
|
|
3262
|
+
}
|
|
3263
|
+
if (action.type === "take_screenshot") {
|
|
3264
|
+
const html2canvasModule = await import("html2canvas");
|
|
3265
|
+
const html2canvas2 = html2canvasModule.default;
|
|
3266
|
+
const canvas = await html2canvas2(document.body, {
|
|
3267
|
+
useCORS: true,
|
|
3268
|
+
allowTaint: true,
|
|
3269
|
+
scale: Math.min(window.devicePixelRatio, 2),
|
|
3270
|
+
width: window.innerWidth,
|
|
3271
|
+
height: window.innerHeight,
|
|
3272
|
+
x: window.scrollX,
|
|
3273
|
+
y: window.scrollY,
|
|
3274
|
+
logging: false
|
|
3275
|
+
});
|
|
3276
|
+
return { result: canvas.toDataURL("image/png") };
|
|
3277
|
+
}
|
|
3278
|
+
if (action.type === "navigate_to_url") {
|
|
3279
|
+
const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
|
|
3280
|
+
if (!nextUrl) {
|
|
3281
|
+
throw new Error("navigate_to_url missing url");
|
|
3313
3282
|
}
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
const isTerminal = isTerminalAction(command);
|
|
3324
|
-
if (isTerminal) {
|
|
3325
|
-
await Promise.all(pendingUIActions);
|
|
3326
|
-
pendingUIActions.length = 0;
|
|
3327
|
-
}
|
|
3328
|
-
const executionPromise = (async () => {
|
|
3329
|
-
const execution = await executeOne(command);
|
|
3330
|
-
await execution.settlePromise;
|
|
3331
|
-
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3332
|
-
})();
|
|
3333
|
-
if (isTerminal) {
|
|
3334
|
-
await executionPromise;
|
|
3335
|
-
} else {
|
|
3336
|
-
pendingUIActions.push(executionPromise);
|
|
3337
|
-
}
|
|
3283
|
+
await navigateToTourUrl(nextUrl);
|
|
3284
|
+
const { waitForDomSettle: waitForDomSettleNav } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3285
|
+
await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3286
|
+
return { result: nextUrl };
|
|
3287
|
+
}
|
|
3288
|
+
if (action.type === "execute_agent_action") {
|
|
3289
|
+
const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
|
|
3290
|
+
if (!agentSocketId) {
|
|
3291
|
+
throw new Error("No socketId available for execute_agent_action");
|
|
3338
3292
|
}
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3293
|
+
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3294
|
+
const res = await fetch(url, {
|
|
3295
|
+
method: "POST",
|
|
3296
|
+
headers: { "Content-Type": "application/json" },
|
|
3297
|
+
body: JSON.stringify({
|
|
3298
|
+
command: action.params?.command,
|
|
3299
|
+
socketId: agentSocketId
|
|
3300
|
+
})
|
|
3342
3301
|
});
|
|
3302
|
+
if (!res.ok) {
|
|
3303
|
+
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3304
|
+
}
|
|
3305
|
+
if (action.params?.wait !== false) {
|
|
3306
|
+
await res.json();
|
|
3307
|
+
}
|
|
3308
|
+
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3309
|
+
await waitForDomSettle({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3343
3310
|
await syncAOM();
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3311
|
+
return { result: action.params?.command ?? "executed" };
|
|
3312
|
+
}
|
|
3313
|
+
if (action.type === "wait_for_user_action") {
|
|
3314
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3315
|
+
if (!targetEl) {
|
|
3316
|
+
throw new Error("wait_for_user_action target not found");
|
|
3317
|
+
}
|
|
3318
|
+
const eventName = action.params?.event ?? "click";
|
|
3319
|
+
await waitForUserAction(targetEl, eventName);
|
|
3320
|
+
return { result: `waited_for_${eventName}` };
|
|
3321
|
+
}
|
|
3322
|
+
if (action.type === "wait_for_input") {
|
|
3323
|
+
shouldWait = true;
|
|
3324
|
+
return { result: "waiting_for_input" };
|
|
3325
|
+
}
|
|
3326
|
+
if (action.type === "end_tour") {
|
|
3327
|
+
handleTourEnd();
|
|
3328
|
+
return { result: "ended" };
|
|
3329
|
+
}
|
|
3330
|
+
console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
|
|
3331
|
+
return { result: "unknown_action_skipped" };
|
|
3332
|
+
};
|
|
3333
|
+
try {
|
|
3334
|
+
const resultsBuffer = new Array(payload.commands.length);
|
|
3335
|
+
const pendingUIActions = [];
|
|
3336
|
+
for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
|
|
3337
|
+
const command = payload.commands[commandIndex];
|
|
3338
|
+
assertNotInterrupted();
|
|
3339
|
+
const isTerminal = isTerminalAction(command);
|
|
3340
|
+
if (isTerminal) {
|
|
3341
|
+
await Promise.all(pendingUIActions);
|
|
3342
|
+
pendingUIActions.length = 0;
|
|
3369
3343
|
}
|
|
3370
|
-
|
|
3344
|
+
const executionPromise = (async () => {
|
|
3345
|
+
const execution = await executeOne(command);
|
|
3346
|
+
await execution.settlePromise;
|
|
3347
|
+
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3348
|
+
})();
|
|
3349
|
+
if (isTerminal) {
|
|
3350
|
+
await executionPromise;
|
|
3351
|
+
} else {
|
|
3352
|
+
pendingUIActions.push(executionPromise);
|
|
3353
|
+
}
|
|
3354
|
+
}
|
|
3355
|
+
await Promise.all(pendingUIActions);
|
|
3356
|
+
resultsBuffer.forEach((res) => {
|
|
3357
|
+
if (res) results.push(res);
|
|
3358
|
+
});
|
|
3359
|
+
await syncAOM();
|
|
3360
|
+
} catch (err) {
|
|
3361
|
+
commandInFlightRef.current = false;
|
|
3362
|
+
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3363
|
+
if (interrupted) {
|
|
3371
3364
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3372
3365
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3373
3366
|
stepOrder: stepIndexRef.current,
|
|
3374
|
-
eventType: "
|
|
3367
|
+
eventType: "command_batch_interrupted",
|
|
3375
3368
|
payload: {
|
|
3376
3369
|
commandBatchId,
|
|
3377
|
-
error: String(err),
|
|
3378
3370
|
partialResults: results
|
|
3379
3371
|
},
|
|
3380
|
-
status: "active",
|
|
3381
3372
|
currentStepOrder: stepIndexRef.current
|
|
3382
3373
|
});
|
|
3383
3374
|
}
|
|
3384
|
-
|
|
3385
|
-
success:
|
|
3386
|
-
|
|
3387
|
-
error: String(err),
|
|
3375
|
+
emitIfOpen("tour:action_result", {
|
|
3376
|
+
success: true,
|
|
3377
|
+
interrupted: true,
|
|
3388
3378
|
results,
|
|
3389
3379
|
commandBatchId,
|
|
3390
3380
|
runId: runIdRef.current,
|
|
@@ -3393,108 +3383,82 @@ function useTourPlayback({
|
|
|
3393
3383
|
clearCommandBatchId();
|
|
3394
3384
|
return;
|
|
3395
3385
|
}
|
|
3396
|
-
|
|
3397
|
-
if (
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
|
|
3386
|
+
console.error("[TourClient] Command batch execution failed:", err);
|
|
3387
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3388
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3389
|
+
stepOrder: stepIndexRef.current,
|
|
3390
|
+
eventType: "command_batch_failed",
|
|
3391
|
+
payload: {
|
|
3392
|
+
commandBatchId,
|
|
3393
|
+
error: String(err),
|
|
3394
|
+
partialResults: results
|
|
3395
|
+
},
|
|
3396
|
+
status: "active",
|
|
3397
|
+
currentStepOrder: stepIndexRef.current
|
|
3398
|
+
});
|
|
3399
|
+
}
|
|
3400
|
+
emitIfOpen("tour:action_result", {
|
|
3401
|
+
success: false,
|
|
3402
|
+
reason: "execution_error",
|
|
3403
|
+
error: String(err),
|
|
3404
|
+
results,
|
|
3405
|
+
commandBatchId,
|
|
3406
|
+
runId: runIdRef.current,
|
|
3407
|
+
turnId: turnIdRef.current
|
|
3408
|
+
});
|
|
3409
|
+
clearCommandBatchId();
|
|
3410
|
+
return;
|
|
3411
|
+
}
|
|
3412
|
+
commandInFlightRef.current = false;
|
|
3413
|
+
if (shouldWait && !skipRequestedRef.current) {
|
|
3414
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3415
|
+
const waitCondition = currentStep?.onboarding?.waitCondition;
|
|
3416
|
+
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3417
|
+
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3418
|
+
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3419
|
+
let manualWaitPromise = null;
|
|
3420
|
+
let manualWaitKind = null;
|
|
3421
|
+
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3422
|
+
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3423
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3424
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3425
|
+
if (waitTargetHints) {
|
|
3426
|
+
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3427
|
+
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3428
|
+
manualWaitTarget = preferredWaitTarget;
|
|
3429
|
+
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3421
3430
|
}
|
|
3422
|
-
if (
|
|
3423
|
-
const manualWait = createManualWaitForTarget(
|
|
3431
|
+
if (manualWaitTarget) {
|
|
3432
|
+
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3424
3433
|
manualWaitPromise = manualWait.promise;
|
|
3425
3434
|
manualWaitKind = manualWait.kind;
|
|
3426
3435
|
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3427
|
-
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3428
3436
|
}
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
commandBatchId,
|
|
3448
|
-
results
|
|
3449
|
-
},
|
|
3450
|
-
currentStepOrder: stepIndexRef.current
|
|
3451
|
-
});
|
|
3437
|
+
}
|
|
3438
|
+
if (!manualWaitPromise && preferredWaitTarget) {
|
|
3439
|
+
const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
|
|
3440
|
+
manualWaitPromise = manualWait.promise;
|
|
3441
|
+
manualWaitKind = manualWait.kind;
|
|
3442
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3443
|
+
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3444
|
+
}
|
|
3445
|
+
if (!manualWaitPromise && inputLikeWait) {
|
|
3446
|
+
const firstInput = document.querySelector(
|
|
3447
|
+
'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
|
|
3448
|
+
);
|
|
3449
|
+
if (firstInput) {
|
|
3450
|
+
const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
|
|
3451
|
+
manualWaitPromise = manualWait.promise;
|
|
3452
|
+
manualWaitKind = manualWait.kind;
|
|
3453
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3454
|
+
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3452
3455
|
}
|
|
3453
|
-
socket.emit("tour:action_result", {
|
|
3454
|
-
success: true,
|
|
3455
|
-
waitingForInput: true,
|
|
3456
|
-
results,
|
|
3457
|
-
commandBatchId,
|
|
3458
|
-
runId: runIdRef.current,
|
|
3459
|
-
turnId: turnIdRef.current
|
|
3460
|
-
});
|
|
3461
|
-
clearCommandBatchId();
|
|
3462
|
-
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3463
|
-
if (pendingInputBufRef.current) {
|
|
3464
|
-
const flushed = pendingInputBufRef.current;
|
|
3465
|
-
pendingInputBufRef.current = null;
|
|
3466
|
-
resolve(flushed);
|
|
3467
|
-
return;
|
|
3468
|
-
}
|
|
3469
|
-
voiceInputResolveRef.current = (text) => {
|
|
3470
|
-
voiceInputResolveRef.current = null;
|
|
3471
|
-
resolve(text);
|
|
3472
|
-
};
|
|
3473
|
-
});
|
|
3474
|
-
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3475
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3476
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3477
|
-
voiceInputResolveRef.current = null;
|
|
3478
|
-
setPlaybackState("executing");
|
|
3479
|
-
const transcript = userText.trim();
|
|
3480
|
-
if (!transcript) {
|
|
3481
|
-
return;
|
|
3482
|
-
}
|
|
3483
|
-
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3484
|
-
await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
|
|
3485
|
-
await syncAOM();
|
|
3486
|
-
socket.emit("tour:user_input", {
|
|
3487
|
-
transcript,
|
|
3488
|
-
runId: runIdRef.current,
|
|
3489
|
-
turnId: turnIdRef.current
|
|
3490
|
-
});
|
|
3491
|
-
});
|
|
3492
|
-
return;
|
|
3493
3456
|
}
|
|
3457
|
+
setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
|
|
3494
3458
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3495
3459
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3496
3460
|
stepOrder: stepIndexRef.current,
|
|
3497
|
-
eventType: "
|
|
3461
|
+
eventType: "waiting_for_input",
|
|
3498
3462
|
payload: {
|
|
3499
3463
|
commandBatchId,
|
|
3500
3464
|
results
|
|
@@ -3502,101 +3466,169 @@ function useTourPlayback({
|
|
|
3502
3466
|
currentStepOrder: stepIndexRef.current
|
|
3503
3467
|
});
|
|
3504
3468
|
}
|
|
3505
|
-
|
|
3469
|
+
emitIfOpen("tour:action_result", {
|
|
3506
3470
|
success: true,
|
|
3471
|
+
waitingForInput: true,
|
|
3507
3472
|
results,
|
|
3508
3473
|
commandBatchId,
|
|
3509
3474
|
runId: runIdRef.current,
|
|
3510
3475
|
turnId: turnIdRef.current
|
|
3511
3476
|
});
|
|
3512
3477
|
clearCommandBatchId();
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
|
|
3540
|
-
|
|
3478
|
+
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3479
|
+
if (pendingInputBufRef.current) {
|
|
3480
|
+
const flushed = pendingInputBufRef.current;
|
|
3481
|
+
pendingInputBufRef.current = null;
|
|
3482
|
+
resolve(flushed);
|
|
3483
|
+
return;
|
|
3484
|
+
}
|
|
3485
|
+
voiceInputResolveRef.current = (text) => {
|
|
3486
|
+
voiceInputResolveRef.current = null;
|
|
3487
|
+
resolve(text);
|
|
3488
|
+
};
|
|
3489
|
+
});
|
|
3490
|
+
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3491
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3492
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3493
|
+
voiceInputResolveRef.current = null;
|
|
3494
|
+
setPlaybackState("executing");
|
|
3495
|
+
const transcript = userText.trim();
|
|
3496
|
+
if (!transcript) {
|
|
3497
|
+
return;
|
|
3498
|
+
}
|
|
3499
|
+
const { waitForDomSettle } = await import("./dom-sync-L5KIP45X.mjs");
|
|
3500
|
+
await waitForDomSettle({ timeoutMs: 1500, debounceMs: 200 });
|
|
3501
|
+
await syncAOM();
|
|
3502
|
+
emitIfOpen("tour:user_input", {
|
|
3503
|
+
transcript,
|
|
3504
|
+
runId: runIdRef.current,
|
|
3505
|
+
turnId: turnIdRef.current
|
|
3541
3506
|
});
|
|
3542
|
-
}
|
|
3543
|
-
|
|
3544
|
-
|
|
3545
|
-
|
|
3507
|
+
});
|
|
3508
|
+
return;
|
|
3509
|
+
}
|
|
3510
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3511
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3512
|
+
stepOrder: stepIndexRef.current,
|
|
3513
|
+
eventType: "command_batch_completed",
|
|
3514
|
+
payload: {
|
|
3515
|
+
commandBatchId,
|
|
3516
|
+
results
|
|
3517
|
+
},
|
|
3518
|
+
currentStepOrder: stepIndexRef.current
|
|
3519
|
+
});
|
|
3520
|
+
}
|
|
3521
|
+
emitIfOpen("tour:action_result", {
|
|
3522
|
+
success: true,
|
|
3523
|
+
results,
|
|
3524
|
+
commandBatchId,
|
|
3525
|
+
runId: runIdRef.current,
|
|
3526
|
+
turnId: turnIdRef.current
|
|
3527
|
+
});
|
|
3528
|
+
clearCommandBatchId();
|
|
3529
|
+
});
|
|
3530
|
+
socket.on("tour:start", async (tourData) => {
|
|
3531
|
+
if (isActiveRef.current) return;
|
|
3532
|
+
runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
|
|
3533
|
+
const tour = tourData.tourContext ?? tourRef.current;
|
|
3534
|
+
const expType = experienceTypeRef.current;
|
|
3535
|
+
if (tour?.type && tour.type !== expType) {
|
|
3536
|
+
console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
|
|
3537
|
+
return;
|
|
3538
|
+
}
|
|
3539
|
+
skipRequestedRef.current = false;
|
|
3540
|
+
const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
|
|
3541
|
+
isActiveRef.current = true;
|
|
3542
|
+
setIsActive(true);
|
|
3543
|
+
setActiveTour(tour ?? null);
|
|
3544
|
+
tourRef.current = tour ?? null;
|
|
3545
|
+
setTotalSteps(total);
|
|
3546
|
+
stepIndexRef.current = 0;
|
|
3547
|
+
setCurrentStepIndex(0);
|
|
3548
|
+
setPlaybackState("intro");
|
|
3549
|
+
if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
|
|
3550
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
|
|
3551
|
+
stepOrder: 0,
|
|
3552
|
+
eventType: "tour_started",
|
|
3553
|
+
payload: {
|
|
3554
|
+
totalSteps: total,
|
|
3555
|
+
source: "sdk_test_preview"
|
|
3556
|
+
},
|
|
3557
|
+
currentStepOrder: 0
|
|
3558
|
+
});
|
|
3559
|
+
}
|
|
3560
|
+
try {
|
|
3561
|
+
const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
|
|
3562
|
+
const aom = generateMinifiedAOM2();
|
|
3563
|
+
if (socketRef.current === socket && socket.connected) {
|
|
3546
3564
|
socket.emit("tour:sync_dom", {
|
|
3547
3565
|
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3548
3566
|
aom: aom.nodes,
|
|
3549
3567
|
domSummary: captureDomSummary()
|
|
3550
3568
|
});
|
|
3551
|
-
} catch (e) {
|
|
3552
|
-
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3553
3569
|
}
|
|
3554
|
-
})
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3570
|
+
} catch (e) {
|
|
3571
|
+
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3572
|
+
}
|
|
3573
|
+
});
|
|
3574
|
+
socket.on("tour:update", (payload) => {
|
|
3575
|
+
const updatedTour = payload?.tourContext;
|
|
3576
|
+
if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
|
|
3577
|
+
return;
|
|
3578
|
+
}
|
|
3579
|
+
tourRef.current = updatedTour;
|
|
3580
|
+
setActiveTour(updatedTour);
|
|
3581
|
+
const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
|
|
3582
|
+
setTotalSteps(nextTotal);
|
|
3583
|
+
if (typeof payload.currentStepIndex === "number") {
|
|
3584
|
+
const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
|
|
3585
|
+
stepIndexRef.current = clampedStepIndex;
|
|
3586
|
+
setCurrentStepIndex(clampedStepIndex);
|
|
3587
|
+
if (nextTotal > 0) {
|
|
3588
|
+
onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
|
|
3571
3589
|
}
|
|
3572
|
-
}
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
}
|
|
3585
|
-
}
|
|
3590
|
+
}
|
|
3591
|
+
});
|
|
3592
|
+
socket.on("tour:end", () => {
|
|
3593
|
+
setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
|
|
3594
|
+
handleTourEnd();
|
|
3595
|
+
});
|
|
3596
|
+
socket.on("tour:debug_log", (entry) => {
|
|
3597
|
+
const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
|
|
3598
|
+
if (isDev) {
|
|
3599
|
+
console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
|
|
3600
|
+
if (typeof window !== "undefined") {
|
|
3601
|
+
window.dispatchEvent(new CustomEvent("modelnex-debug", {
|
|
3602
|
+
detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
|
|
3603
|
+
}));
|
|
3586
3604
|
}
|
|
3587
|
-
}
|
|
3588
|
-
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devMode || process.env.NODE_ENV === "development");
|
|
3605
|
+
}
|
|
3589
3606
|
});
|
|
3607
|
+
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3590
3608
|
return () => {
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3609
|
+
const toClose = createdSocket ?? socketRef.current;
|
|
3610
|
+
if (toClose) {
|
|
3611
|
+
toClose.disconnect();
|
|
3594
3612
|
}
|
|
3613
|
+
createdSocket = null;
|
|
3614
|
+
socketRef.current = null;
|
|
3595
3615
|
setServerState(null);
|
|
3596
3616
|
runIdRef.current = null;
|
|
3597
3617
|
turnIdRef.current = null;
|
|
3598
3618
|
};
|
|
3599
|
-
}, [serverUrl, websiteId,
|
|
3619
|
+
}, [serverUrl, websiteId, disabled]);
|
|
3620
|
+
useEffect11(() => {
|
|
3621
|
+
if (disabled) return;
|
|
3622
|
+
const s = socketRef.current;
|
|
3623
|
+
const profile = userProfile;
|
|
3624
|
+
if (!s?.connected || !websiteId || !profile?.userId) return;
|
|
3625
|
+
const timer = setTimeout(() => {
|
|
3626
|
+
if (s.connected) {
|
|
3627
|
+
s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3628
|
+
}
|
|
3629
|
+
}, 150);
|
|
3630
|
+
return () => clearTimeout(timer);
|
|
3631
|
+
}, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
|
|
3600
3632
|
useEffect11(() => {
|
|
3601
3633
|
if (!showCaptions || !isReviewMode) {
|
|
3602
3634
|
removeCaption();
|
|
@@ -3620,11 +3652,13 @@ function useTourPlayback({
|
|
|
3620
3652
|
if (!socketRef.current?.connected || !isActiveRef.current) return;
|
|
3621
3653
|
const { generateMinifiedAOM: generateMinifiedAOM2 } = await import("./aom-HDYNCIOY.mjs");
|
|
3622
3654
|
const aom = generateMinifiedAOM2();
|
|
3623
|
-
socketRef.current
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3655
|
+
if (socketRef.current?.connected) {
|
|
3656
|
+
socketRef.current.emit("tour:sync_dom", {
|
|
3657
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3658
|
+
aom: aom.nodes,
|
|
3659
|
+
domSummary: captureDomSummary()
|
|
3660
|
+
});
|
|
3661
|
+
}
|
|
3628
3662
|
}, []);
|
|
3629
3663
|
const interruptExecution = useCallback7((transcript) => {
|
|
3630
3664
|
if (!socketRef.current?.connected || !isActiveRef.current) return false;
|
|
@@ -3635,21 +3669,23 @@ function useTourPlayback({
|
|
|
3635
3669
|
removeHighlight();
|
|
3636
3670
|
removeCaption();
|
|
3637
3671
|
voice.stopSpeaking();
|
|
3638
|
-
socketRef.current
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3672
|
+
if (socketRef.current?.connected) {
|
|
3673
|
+
socketRef.current.emit("tour:action_result", {
|
|
3674
|
+
success: true,
|
|
3675
|
+
interrupted: true,
|
|
3676
|
+
transcript,
|
|
3677
|
+
commandBatchId: activeCommandBatchIdRef.current,
|
|
3678
|
+
runId: runIdRef.current,
|
|
3679
|
+
turnId: turnIdRef.current
|
|
3680
|
+
});
|
|
3681
|
+
activeCommandBatchIdRef.current = null;
|
|
3682
|
+
socketRef.current.emit("tour:user_input", {
|
|
3683
|
+
transcript,
|
|
3684
|
+
interrupted: true,
|
|
3685
|
+
runId: runIdRef.current,
|
|
3686
|
+
turnId: turnIdRef.current
|
|
3687
|
+
});
|
|
3688
|
+
}
|
|
3653
3689
|
setPlaybackState("thinking");
|
|
3654
3690
|
return true;
|
|
3655
3691
|
}, [voice]);
|
|
@@ -3767,11 +3803,13 @@ function useTourPlayback({
|
|
|
3767
3803
|
previewRunIdRef.current = null;
|
|
3768
3804
|
}
|
|
3769
3805
|
tourRef.current = tour;
|
|
3770
|
-
socketRef.current
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3806
|
+
if (socketRef.current?.connected) {
|
|
3807
|
+
socketRef.current.emit("tour:request_start", {
|
|
3808
|
+
tourId: tour.id,
|
|
3809
|
+
previewRunId: previewRunIdRef.current,
|
|
3810
|
+
tourContext: tour
|
|
3811
|
+
});
|
|
3812
|
+
}
|
|
3775
3813
|
}, [serverUrl, websiteId]);
|
|
3776
3814
|
useEffect11(() => {
|
|
3777
3815
|
if (!enableAutoDiscovery) return;
|
|
@@ -5219,7 +5257,7 @@ function useVoice(serverUrl) {
|
|
|
5219
5257
|
(async () => {
|
|
5220
5258
|
try {
|
|
5221
5259
|
const ioModule = await import("socket.io-client");
|
|
5222
|
-
const
|
|
5260
|
+
const io3 = ioModule.default || ioModule;
|
|
5223
5261
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
5224
5262
|
audio: {
|
|
5225
5263
|
echoCancellation: true,
|
|
@@ -5241,7 +5279,7 @@ function useVoice(serverUrl) {
|
|
|
5241
5279
|
audioBitsPerSecond: 128e3
|
|
5242
5280
|
});
|
|
5243
5281
|
mediaRecorderRef.current = recorder;
|
|
5244
|
-
const socket =
|
|
5282
|
+
const socket = io3(serverUrl, {
|
|
5245
5283
|
path: "/socket.io",
|
|
5246
5284
|
transports: resolveSocketIoTransports(serverUrl, "websocket-first")
|
|
5247
5285
|
});
|
|
@@ -10761,7 +10799,7 @@ var ModelNexProvider = ({
|
|
|
10761
10799
|
socketId,
|
|
10762
10800
|
devMode
|
|
10763
10801
|
}),
|
|
10764
|
-
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
|
|
10802
|
+
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
|
|
10765
10803
|
);
|
|
10766
10804
|
return React8.createElement(
|
|
10767
10805
|
ModelNexContext.Provider,
|