@modelnex/sdk 0.5.14 → 0.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +589 -551
- package/dist/index.mjs +589 -551
- package/package.json +11 -12
- package/dist/aom-GA6W42DG.mjs +0 -71
- package/dist/aom-J6NYMGDW.mjs +0 -69
- package/dist/chunk-6DZX6EAA.mjs +0 -37
- package/dist/dom-sync-Y7Z7TOU6.mjs +0 -57
package/dist/index.js
CHANGED
|
@@ -2324,7 +2324,7 @@ function useBuiltinActions(registerAction, unregisterAction, tagStore, serverUrl
|
|
|
2324
2324
|
}
|
|
2325
2325
|
|
|
2326
2326
|
// src/constants.ts
|
|
2327
|
-
var DEFAULT_MODELNEX_SERVER_URL = "https://modelnex
|
|
2327
|
+
var DEFAULT_MODELNEX_SERVER_URL = "https://api.modelnex.com";
|
|
2328
2328
|
|
|
2329
2329
|
// src/hooks/useRunCommand.ts
|
|
2330
2330
|
var import_react9 = require("react");
|
|
@@ -2626,6 +2626,7 @@ function isTourEligible(tour, userProfile) {
|
|
|
2626
2626
|
|
|
2627
2627
|
// src/hooks/useTourPlayback.ts
|
|
2628
2628
|
var import_react12 = require("react");
|
|
2629
|
+
var import_socket2 = require("socket.io-client");
|
|
2629
2630
|
|
|
2630
2631
|
// src/utils/retryLookup.ts
|
|
2631
2632
|
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -3154,6 +3155,12 @@ function useTourPlayback({
|
|
|
3154
3155
|
const [serverState, setServerState] = (0, import_react12.useState)(null);
|
|
3155
3156
|
const ctx = (0, import_react12.useContext)(ModelNexContext);
|
|
3156
3157
|
const devMode = ctx?.devMode;
|
|
3158
|
+
const devModeRef = (0, import_react12.useRef)(devMode);
|
|
3159
|
+
devModeRef.current = devMode;
|
|
3160
|
+
const userProfileRef = (0, import_react12.useRef)(userProfile);
|
|
3161
|
+
userProfileRef.current = userProfile;
|
|
3162
|
+
const experienceTypeRef = (0, import_react12.useRef)(experienceType);
|
|
3163
|
+
experienceTypeRef.current = experienceType;
|
|
3157
3164
|
const tourRef = (0, import_react12.useRef)(null);
|
|
3158
3165
|
const stepIndexRef = (0, import_react12.useRef)(0);
|
|
3159
3166
|
const skipRequestedRef = (0, import_react12.useRef)(false);
|
|
@@ -3193,408 +3200,391 @@ function useTourPlayback({
|
|
|
3193
3200
|
(0, import_react12.useEffect)(() => {
|
|
3194
3201
|
if (disabled) return;
|
|
3195
3202
|
if (typeof window === "undefined") return;
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
socket.
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3203
|
+
let createdSocket = null;
|
|
3204
|
+
const socket = (0, import_socket2.io)(serverUrl, {
|
|
3205
|
+
path: "/socket.io",
|
|
3206
|
+
transports: resolveSocketIoTransports(serverUrl, "polling-first"),
|
|
3207
|
+
autoConnect: true,
|
|
3208
|
+
reconnection: true,
|
|
3209
|
+
reconnectionAttempts: 10,
|
|
3210
|
+
reconnectionDelay: 1e3
|
|
3211
|
+
});
|
|
3212
|
+
createdSocket = socket;
|
|
3213
|
+
socketRef.current = socket;
|
|
3214
|
+
socket.on("connect", () => {
|
|
3215
|
+
console.log("[TourClient] Connected to tour agent server:", socket.id);
|
|
3216
|
+
const profile = userProfileRef.current;
|
|
3217
|
+
if (websiteId && profile?.userId && socket.connected) {
|
|
3218
|
+
socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3219
|
+
}
|
|
3220
|
+
});
|
|
3221
|
+
socket.on("tour:server_state", (payload) => {
|
|
3222
|
+
if (typeof payload?.runId === "number") {
|
|
3223
|
+
runIdRef.current = payload.runId;
|
|
3224
|
+
}
|
|
3225
|
+
if (typeof payload?.turnId === "string" || payload?.turnId === null) {
|
|
3226
|
+
turnIdRef.current = payload.turnId ?? null;
|
|
3227
|
+
}
|
|
3228
|
+
setServerState(payload);
|
|
3229
|
+
});
|
|
3230
|
+
socket.on("tour:command_cancel", (payload) => {
|
|
3231
|
+
console.log("[TourClient] Received command_cancel:", payload);
|
|
3232
|
+
if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
|
|
3233
|
+
activeCommandBatchIdRef.current = null;
|
|
3234
|
+
activeExecutionTokenRef.current++;
|
|
3235
|
+
commandInFlightRef.current = false;
|
|
3236
|
+
setPlaybackState("idle");
|
|
3237
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3238
|
+
window.speechSynthesis.cancel();
|
|
3216
3239
|
}
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
if (
|
|
3240
|
+
}
|
|
3241
|
+
});
|
|
3242
|
+
socket.on("tour:command", async (payload) => {
|
|
3243
|
+
const emitIfOpen = (ev, data) => {
|
|
3244
|
+
if (socketRef.current !== socket) return;
|
|
3245
|
+
if (!socket.connected) return;
|
|
3246
|
+
socket.emit(ev, data);
|
|
3247
|
+
};
|
|
3248
|
+
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3249
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3250
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3251
|
+
if (voiceInputResolveRef.current) {
|
|
3252
|
+
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3253
|
+
voiceInputResolveRef.current = null;
|
|
3254
|
+
resolvePendingVoiceInput("");
|
|
3255
|
+
}
|
|
3256
|
+
setPlaybackState("executing");
|
|
3257
|
+
commandInFlightRef.current = true;
|
|
3258
|
+
const commandBatchId = payload.commandBatchId ?? null;
|
|
3259
|
+
turnIdRef.current = payload.turnId ?? turnIdRef.current;
|
|
3260
|
+
const clearCommandBatchId = () => {
|
|
3261
|
+
if (activeCommandBatchIdRef.current === commandBatchId) {
|
|
3222
3262
|
activeCommandBatchIdRef.current = null;
|
|
3223
|
-
activeExecutionTokenRef.current++;
|
|
3224
|
-
commandInFlightRef.current = false;
|
|
3225
|
-
setPlaybackState("idle");
|
|
3226
|
-
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3227
|
-
window.speechSynthesis.cancel();
|
|
3228
|
-
}
|
|
3229
3263
|
}
|
|
3230
|
-
}
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3264
|
+
};
|
|
3265
|
+
activeCommandBatchIdRef.current = commandBatchId;
|
|
3266
|
+
const executionToken = ++activeExecutionTokenRef.current;
|
|
3267
|
+
const activeTourId = tourRef.current?.id;
|
|
3268
|
+
const activePreviewRunId = previewRunIdRef.current;
|
|
3269
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3270
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3271
|
+
stepOrder: payload.stepIndex,
|
|
3272
|
+
eventType: "command_batch_received",
|
|
3273
|
+
payload: {
|
|
3274
|
+
commandBatchId,
|
|
3275
|
+
commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
|
|
3276
|
+
},
|
|
3277
|
+
currentStepOrder: payload.stepIndex
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
if (typeof payload.stepIndex === "number") {
|
|
3281
|
+
const prevStep = stepIndexRef.current;
|
|
3282
|
+
stepIndexRef.current = payload.stepIndex;
|
|
3283
|
+
setCurrentStepIndex(payload.stepIndex);
|
|
3284
|
+
if (payload.stepIndex !== prevStep) {
|
|
3285
|
+
const tour = tourRef.current;
|
|
3286
|
+
const total = tour?.steps?.length ?? 0;
|
|
3287
|
+
if (tour && total > 0) {
|
|
3288
|
+
onStepChangeRef.current?.(payload.stepIndex, total, tour);
|
|
3247
3289
|
}
|
|
3248
|
-
}
|
|
3249
|
-
|
|
3250
|
-
const executionToken = ++activeExecutionTokenRef.current;
|
|
3251
|
-
const activeTourId = tourRef.current?.id;
|
|
3252
|
-
const activePreviewRunId = previewRunIdRef.current;
|
|
3253
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3290
|
+
}
|
|
3291
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3254
3292
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3255
3293
|
stepOrder: payload.stepIndex,
|
|
3256
|
-
eventType: "
|
|
3294
|
+
eventType: "step_started",
|
|
3257
3295
|
payload: {
|
|
3258
|
-
|
|
3259
|
-
|
|
3296
|
+
previousStepOrder: prevStep,
|
|
3297
|
+
stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
|
|
3260
3298
|
},
|
|
3261
3299
|
currentStepOrder: payload.stepIndex
|
|
3262
3300
|
});
|
|
3263
3301
|
}
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
}
|
|
3287
|
-
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3288
|
-
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3289
|
-
commandInFlightRef.current = false;
|
|
3290
|
-
socket.emit("tour:action_result", {
|
|
3291
|
-
success: false,
|
|
3292
|
-
reason: "invalid_commands",
|
|
3293
|
-
commandBatchId,
|
|
3294
|
-
runId: runIdRef.current,
|
|
3295
|
-
turnId: turnIdRef.current
|
|
3296
|
-
});
|
|
3297
|
-
clearCommandBatchId();
|
|
3298
|
-
return;
|
|
3302
|
+
}
|
|
3303
|
+
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3304
|
+
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3305
|
+
commandInFlightRef.current = false;
|
|
3306
|
+
emitIfOpen("tour:action_result", {
|
|
3307
|
+
success: false,
|
|
3308
|
+
reason: "invalid_commands",
|
|
3309
|
+
commandBatchId,
|
|
3310
|
+
runId: runIdRef.current,
|
|
3311
|
+
turnId: turnIdRef.current
|
|
3312
|
+
});
|
|
3313
|
+
clearCommandBatchId();
|
|
3314
|
+
return;
|
|
3315
|
+
}
|
|
3316
|
+
let shouldWait = false;
|
|
3317
|
+
const results = [];
|
|
3318
|
+
let batchPreferredWaitTarget = null;
|
|
3319
|
+
const assertNotInterrupted = () => {
|
|
3320
|
+
if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
|
|
3321
|
+
const error = new Error("interrupted");
|
|
3322
|
+
error.code = "INTERRUPTED";
|
|
3323
|
+
throw error;
|
|
3299
3324
|
}
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
}
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3315
|
-
onRetry: () => {
|
|
3316
|
-
assertNotInterrupted();
|
|
3317
|
-
},
|
|
3318
|
-
resolve: async () => {
|
|
3319
|
-
let targetEl = null;
|
|
3320
|
-
if (params.uid) {
|
|
3321
|
-
const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3322
|
-
targetEl = getElementByUid2(params.uid);
|
|
3323
|
-
}
|
|
3324
|
-
if (!targetEl) {
|
|
3325
|
-
targetEl = resolveElementFromHints({
|
|
3326
|
-
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3327
|
-
testId: params.testId ?? fallbackHints?.testId,
|
|
3328
|
-
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3329
|
-
}, fallbackStep, tagStore);
|
|
3330
|
-
}
|
|
3331
|
-
return targetEl;
|
|
3332
|
-
}
|
|
3333
|
-
});
|
|
3334
|
-
};
|
|
3335
|
-
const executeOne = async (action) => {
|
|
3336
|
-
assertNotInterrupted();
|
|
3337
|
-
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3338
|
-
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3339
|
-
const executeTimeline = async (params = {}) => {
|
|
3340
|
-
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3341
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
3342
|
-
assertNotInterrupted();
|
|
3343
|
-
const segment = segments[index];
|
|
3344
|
-
const segmentText = (segment?.text ?? "").trim();
|
|
3345
|
-
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3346
|
-
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3347
|
-
if (segmentDelayMs > 0) {
|
|
3348
|
-
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3349
|
-
}
|
|
3350
|
-
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3351
|
-
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3352
|
-
if (!gateTarget) {
|
|
3353
|
-
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3354
|
-
}
|
|
3355
|
-
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3356
|
-
}
|
|
3357
|
-
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3358
|
-
showCaption(segmentText);
|
|
3359
|
-
}
|
|
3360
|
-
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3361
|
-
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3362
|
-
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3363
|
-
onNearEnd: nextSegmentText ? () => {
|
|
3364
|
-
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3365
|
-
} : void 0
|
|
3366
|
-
}) : Promise.resolve();
|
|
3367
|
-
const eventsPromise = (async () => {
|
|
3368
|
-
for (const event of events) {
|
|
3369
|
-
assertNotInterrupted();
|
|
3370
|
-
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3371
|
-
if (delayMs > 0) {
|
|
3372
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3373
|
-
}
|
|
3374
|
-
if (event?.action) {
|
|
3375
|
-
await executeOne(event.action);
|
|
3376
|
-
}
|
|
3377
|
-
}
|
|
3378
|
-
})();
|
|
3379
|
-
await Promise.all([speechPromise, eventsPromise]);
|
|
3380
|
-
}
|
|
3381
|
-
if (params.removeHighlightAtEnd !== false) {
|
|
3382
|
-
removeHighlight();
|
|
3383
|
-
}
|
|
3384
|
-
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3385
|
-
removeCaption();
|
|
3386
|
-
}
|
|
3387
|
-
return !!params.waitForInput;
|
|
3388
|
-
};
|
|
3389
|
-
if (action.type === "speak") {
|
|
3390
|
-
const text = action.params?.text ?? "";
|
|
3391
|
-
if (!text.trim()) {
|
|
3392
|
-
return { result: null };
|
|
3325
|
+
};
|
|
3326
|
+
const resolveTargetElement2 = async (params = {}, fallbackStep) => {
|
|
3327
|
+
const fallbackHints = fallbackStep?.element ?? null;
|
|
3328
|
+
return retryLookup({
|
|
3329
|
+
timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
|
|
3330
|
+
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3331
|
+
onRetry: () => {
|
|
3332
|
+
assertNotInterrupted();
|
|
3333
|
+
},
|
|
3334
|
+
resolve: async () => {
|
|
3335
|
+
let targetEl = null;
|
|
3336
|
+
if (params.uid) {
|
|
3337
|
+
const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3338
|
+
targetEl = getElementByUid2(params.uid);
|
|
3393
3339
|
}
|
|
3394
|
-
if (
|
|
3395
|
-
|
|
3340
|
+
if (!targetEl) {
|
|
3341
|
+
targetEl = resolveElementFromHints({
|
|
3342
|
+
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3343
|
+
testId: params.testId ?? fallbackHints?.testId,
|
|
3344
|
+
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3345
|
+
}, fallbackStep, tagStore);
|
|
3396
3346
|
}
|
|
3397
|
-
|
|
3398
|
-
// Schedule narration immediately so the client keeps its speculative,
|
|
3399
|
-
// low-latency behavior, but defer batch completion until playback settles.
|
|
3400
|
-
waitForCompletion: false,
|
|
3401
|
-
interrupt: action.params?.interrupt
|
|
3402
|
-
});
|
|
3403
|
-
return { result: text, settlePromise };
|
|
3347
|
+
return targetEl;
|
|
3404
3348
|
}
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3349
|
+
});
|
|
3350
|
+
};
|
|
3351
|
+
const executeOne = async (action) => {
|
|
3352
|
+
assertNotInterrupted();
|
|
3353
|
+
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3354
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3355
|
+
const executeTimeline = async (params = {}) => {
|
|
3356
|
+
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3357
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
3358
|
+
assertNotInterrupted();
|
|
3359
|
+
const segment = segments[index];
|
|
3360
|
+
const segmentText = (segment?.text ?? "").trim();
|
|
3361
|
+
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3362
|
+
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3363
|
+
if (segmentDelayMs > 0) {
|
|
3364
|
+
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3365
|
+
}
|
|
3366
|
+
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3367
|
+
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3368
|
+
if (!gateTarget) {
|
|
3369
|
+
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3415
3370
|
}
|
|
3416
|
-
|
|
3417
|
-
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3418
|
-
return { result: "highlighted" };
|
|
3371
|
+
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3419
3372
|
}
|
|
3420
|
-
if (
|
|
3421
|
-
|
|
3422
|
-
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3423
|
-
);
|
|
3373
|
+
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3374
|
+
showCaption(segmentText);
|
|
3424
3375
|
}
|
|
3425
|
-
|
|
3376
|
+
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3377
|
+
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3378
|
+
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3379
|
+
onNearEnd: nextSegmentText ? () => {
|
|
3380
|
+
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3381
|
+
} : void 0
|
|
3382
|
+
}) : Promise.resolve();
|
|
3383
|
+
const eventsPromise = (async () => {
|
|
3384
|
+
for (const event of events) {
|
|
3385
|
+
assertNotInterrupted();
|
|
3386
|
+
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3387
|
+
if (delayMs > 0) {
|
|
3388
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3389
|
+
}
|
|
3390
|
+
if (event?.action) {
|
|
3391
|
+
await executeOne(event.action);
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
})();
|
|
3395
|
+
await Promise.all([speechPromise, eventsPromise]);
|
|
3426
3396
|
}
|
|
3427
|
-
if (
|
|
3397
|
+
if (params.removeHighlightAtEnd !== false) {
|
|
3428
3398
|
removeHighlight();
|
|
3429
|
-
return { result: "highlight_removed" };
|
|
3430
3399
|
}
|
|
3431
|
-
if (
|
|
3432
|
-
|
|
3433
|
-
if (!targetEl) {
|
|
3434
|
-
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3435
|
-
throw new Error(
|
|
3436
|
-
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3437
|
-
);
|
|
3438
|
-
}
|
|
3439
|
-
removeHighlight();
|
|
3440
|
-
await performInteractiveClick(targetEl);
|
|
3441
|
-
const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3442
|
-
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3443
|
-
return { result: "clicked" };
|
|
3400
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3401
|
+
removeCaption();
|
|
3444
3402
|
}
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
}
|
|
3452
|
-
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3453
|
-
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3454
|
-
await performInteractiveFill(targetEl, value);
|
|
3455
|
-
return { result: value };
|
|
3403
|
+
return !!params.waitForInput;
|
|
3404
|
+
};
|
|
3405
|
+
if (action.type === "speak") {
|
|
3406
|
+
const text = action.params?.text ?? "";
|
|
3407
|
+
if (!text.trim()) {
|
|
3408
|
+
return { result: null };
|
|
3456
3409
|
}
|
|
3457
|
-
if (
|
|
3458
|
-
|
|
3459
|
-
const html2canvas2 = html2canvasModule.default;
|
|
3460
|
-
const canvas = await html2canvas2(document.body, {
|
|
3461
|
-
useCORS: true,
|
|
3462
|
-
allowTaint: true,
|
|
3463
|
-
scale: Math.min(window.devicePixelRatio, 2),
|
|
3464
|
-
width: window.innerWidth,
|
|
3465
|
-
height: window.innerHeight,
|
|
3466
|
-
x: window.scrollX,
|
|
3467
|
-
y: window.scrollY,
|
|
3468
|
-
logging: false
|
|
3469
|
-
});
|
|
3470
|
-
return { result: canvas.toDataURL("image/png") };
|
|
3410
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3411
|
+
showCaption(text);
|
|
3471
3412
|
}
|
|
3472
|
-
|
|
3473
|
-
|
|
3474
|
-
|
|
3475
|
-
|
|
3413
|
+
const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
|
|
3414
|
+
// Schedule narration immediately so the client keeps its speculative,
|
|
3415
|
+
// low-latency behavior, but defer batch completion until playback settles.
|
|
3416
|
+
waitForCompletion: false,
|
|
3417
|
+
interrupt: action.params?.interrupt
|
|
3418
|
+
});
|
|
3419
|
+
return { result: text, settlePromise };
|
|
3420
|
+
}
|
|
3421
|
+
if (action.type === "play_timeline") {
|
|
3422
|
+
const timelineShouldWait = await executeTimeline(action.params);
|
|
3423
|
+
if (timelineShouldWait) shouldWait = true;
|
|
3424
|
+
return { result: "timeline_executed" };
|
|
3425
|
+
}
|
|
3426
|
+
if (action.type === "highlight_element") {
|
|
3427
|
+
const resolvedEl = await resolveTargetElement2(action.params, currentStep);
|
|
3428
|
+
if (resolvedEl) {
|
|
3429
|
+
if (isEditableWaitTarget(resolvedEl)) {
|
|
3430
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
|
|
3476
3431
|
}
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
|
-
|
|
3480
|
-
return { result: nextUrl };
|
|
3432
|
+
showHighlight(resolvedEl, action.params?.label || action.label);
|
|
3433
|
+
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3434
|
+
return { result: "highlighted" };
|
|
3481
3435
|
}
|
|
3482
|
-
if (action.
|
|
3483
|
-
|
|
3484
|
-
|
|
3485
|
-
|
|
3486
|
-
}
|
|
3487
|
-
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3488
|
-
const res = await fetch(url, {
|
|
3489
|
-
method: "POST",
|
|
3490
|
-
headers: { "Content-Type": "application/json" },
|
|
3491
|
-
body: JSON.stringify({
|
|
3492
|
-
command: action.params?.command,
|
|
3493
|
-
socketId: agentSocketId
|
|
3494
|
-
})
|
|
3495
|
-
});
|
|
3496
|
-
if (!res.ok) {
|
|
3497
|
-
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3498
|
-
}
|
|
3499
|
-
if (action.params?.wait !== false) {
|
|
3500
|
-
await res.json();
|
|
3501
|
-
}
|
|
3502
|
-
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3503
|
-
await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3504
|
-
await syncAOM();
|
|
3505
|
-
return { result: action.params?.command ?? "executed" };
|
|
3436
|
+
if (!action.params?.optional) {
|
|
3437
|
+
throw new Error(
|
|
3438
|
+
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3439
|
+
);
|
|
3506
3440
|
}
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3441
|
+
return { result: "highlight_optional_miss" };
|
|
3442
|
+
}
|
|
3443
|
+
if (action.type === "remove_highlight") {
|
|
3444
|
+
removeHighlight();
|
|
3445
|
+
return { result: "highlight_removed" };
|
|
3446
|
+
}
|
|
3447
|
+
if (action.type === "click_element") {
|
|
3448
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3449
|
+
if (!targetEl) {
|
|
3450
|
+
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3451
|
+
throw new Error(
|
|
3452
|
+
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3453
|
+
);
|
|
3515
3454
|
}
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3455
|
+
removeHighlight();
|
|
3456
|
+
await performInteractiveClick(targetEl);
|
|
3457
|
+
const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3458
|
+
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3459
|
+
return { result: "clicked" };
|
|
3460
|
+
}
|
|
3461
|
+
if (action.type === "fill_input") {
|
|
3462
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3463
|
+
if (!targetEl) {
|
|
3464
|
+
throw new Error(
|
|
3465
|
+
`fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3466
|
+
);
|
|
3519
3467
|
}
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3468
|
+
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3469
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3470
|
+
await performInteractiveFill(targetEl, value);
|
|
3471
|
+
return { result: value };
|
|
3472
|
+
}
|
|
3473
|
+
if (action.type === "take_screenshot") {
|
|
3474
|
+
const html2canvasModule = await import("html2canvas");
|
|
3475
|
+
const html2canvas2 = html2canvasModule.default;
|
|
3476
|
+
const canvas = await html2canvas2(document.body, {
|
|
3477
|
+
useCORS: true,
|
|
3478
|
+
allowTaint: true,
|
|
3479
|
+
scale: Math.min(window.devicePixelRatio, 2),
|
|
3480
|
+
width: window.innerWidth,
|
|
3481
|
+
height: window.innerHeight,
|
|
3482
|
+
x: window.scrollX,
|
|
3483
|
+
y: window.scrollY,
|
|
3484
|
+
logging: false
|
|
3485
|
+
});
|
|
3486
|
+
return { result: canvas.toDataURL("image/png") };
|
|
3487
|
+
}
|
|
3488
|
+
if (action.type === "navigate_to_url") {
|
|
3489
|
+
const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
|
|
3490
|
+
if (!nextUrl) {
|
|
3491
|
+
throw new Error("navigate_to_url missing url");
|
|
3523
3492
|
}
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
const isTerminal = isTerminalAction(command);
|
|
3534
|
-
if (isTerminal) {
|
|
3535
|
-
await Promise.all(pendingUIActions);
|
|
3536
|
-
pendingUIActions.length = 0;
|
|
3537
|
-
}
|
|
3538
|
-
const executionPromise = (async () => {
|
|
3539
|
-
const execution = await executeOne(command);
|
|
3540
|
-
await execution.settlePromise;
|
|
3541
|
-
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3542
|
-
})();
|
|
3543
|
-
if (isTerminal) {
|
|
3544
|
-
await executionPromise;
|
|
3545
|
-
} else {
|
|
3546
|
-
pendingUIActions.push(executionPromise);
|
|
3547
|
-
}
|
|
3493
|
+
await navigateToTourUrl(nextUrl);
|
|
3494
|
+
const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3495
|
+
await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3496
|
+
return { result: nextUrl };
|
|
3497
|
+
}
|
|
3498
|
+
if (action.type === "execute_agent_action") {
|
|
3499
|
+
const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
|
|
3500
|
+
if (!agentSocketId) {
|
|
3501
|
+
throw new Error("No socketId available for execute_agent_action");
|
|
3548
3502
|
}
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3503
|
+
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3504
|
+
const res = await fetch(url, {
|
|
3505
|
+
method: "POST",
|
|
3506
|
+
headers: { "Content-Type": "application/json" },
|
|
3507
|
+
body: JSON.stringify({
|
|
3508
|
+
command: action.params?.command,
|
|
3509
|
+
socketId: agentSocketId
|
|
3510
|
+
})
|
|
3552
3511
|
});
|
|
3512
|
+
if (!res.ok) {
|
|
3513
|
+
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3514
|
+
}
|
|
3515
|
+
if (action.params?.wait !== false) {
|
|
3516
|
+
await res.json();
|
|
3517
|
+
}
|
|
3518
|
+
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3519
|
+
await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3553
3520
|
await syncAOM();
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3521
|
+
return { result: action.params?.command ?? "executed" };
|
|
3522
|
+
}
|
|
3523
|
+
if (action.type === "wait_for_user_action") {
|
|
3524
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3525
|
+
if (!targetEl) {
|
|
3526
|
+
throw new Error("wait_for_user_action target not found");
|
|
3527
|
+
}
|
|
3528
|
+
const eventName = action.params?.event ?? "click";
|
|
3529
|
+
await waitForUserAction(targetEl, eventName);
|
|
3530
|
+
return { result: `waited_for_${eventName}` };
|
|
3531
|
+
}
|
|
3532
|
+
if (action.type === "wait_for_input") {
|
|
3533
|
+
shouldWait = true;
|
|
3534
|
+
return { result: "waiting_for_input" };
|
|
3535
|
+
}
|
|
3536
|
+
if (action.type === "end_tour") {
|
|
3537
|
+
handleTourEnd();
|
|
3538
|
+
return { result: "ended" };
|
|
3539
|
+
}
|
|
3540
|
+
console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
|
|
3541
|
+
return { result: "unknown_action_skipped" };
|
|
3542
|
+
};
|
|
3543
|
+
try {
|
|
3544
|
+
const resultsBuffer = new Array(payload.commands.length);
|
|
3545
|
+
const pendingUIActions = [];
|
|
3546
|
+
for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
|
|
3547
|
+
const command = payload.commands[commandIndex];
|
|
3548
|
+
assertNotInterrupted();
|
|
3549
|
+
const isTerminal = isTerminalAction(command);
|
|
3550
|
+
if (isTerminal) {
|
|
3551
|
+
await Promise.all(pendingUIActions);
|
|
3552
|
+
pendingUIActions.length = 0;
|
|
3579
3553
|
}
|
|
3580
|
-
|
|
3554
|
+
const executionPromise = (async () => {
|
|
3555
|
+
const execution = await executeOne(command);
|
|
3556
|
+
await execution.settlePromise;
|
|
3557
|
+
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3558
|
+
})();
|
|
3559
|
+
if (isTerminal) {
|
|
3560
|
+
await executionPromise;
|
|
3561
|
+
} else {
|
|
3562
|
+
pendingUIActions.push(executionPromise);
|
|
3563
|
+
}
|
|
3564
|
+
}
|
|
3565
|
+
await Promise.all(pendingUIActions);
|
|
3566
|
+
resultsBuffer.forEach((res) => {
|
|
3567
|
+
if (res) results.push(res);
|
|
3568
|
+
});
|
|
3569
|
+
await syncAOM();
|
|
3570
|
+
} catch (err) {
|
|
3571
|
+
commandInFlightRef.current = false;
|
|
3572
|
+
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3573
|
+
if (interrupted) {
|
|
3581
3574
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3582
3575
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3583
3576
|
stepOrder: stepIndexRef.current,
|
|
3584
|
-
eventType: "
|
|
3577
|
+
eventType: "command_batch_interrupted",
|
|
3585
3578
|
payload: {
|
|
3586
3579
|
commandBatchId,
|
|
3587
|
-
error: String(err),
|
|
3588
3580
|
partialResults: results
|
|
3589
3581
|
},
|
|
3590
|
-
status: "active",
|
|
3591
3582
|
currentStepOrder: stepIndexRef.current
|
|
3592
3583
|
});
|
|
3593
3584
|
}
|
|
3594
|
-
|
|
3595
|
-
success:
|
|
3596
|
-
|
|
3597
|
-
error: String(err),
|
|
3585
|
+
emitIfOpen("tour:action_result", {
|
|
3586
|
+
success: true,
|
|
3587
|
+
interrupted: true,
|
|
3598
3588
|
results,
|
|
3599
3589
|
commandBatchId,
|
|
3600
3590
|
runId: runIdRef.current,
|
|
@@ -3603,108 +3593,82 @@ function useTourPlayback({
|
|
|
3603
3593
|
clearCommandBatchId();
|
|
3604
3594
|
return;
|
|
3605
3595
|
}
|
|
3606
|
-
|
|
3607
|
-
if (
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3596
|
+
console.error("[TourClient] Command batch execution failed:", err);
|
|
3597
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3598
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3599
|
+
stepOrder: stepIndexRef.current,
|
|
3600
|
+
eventType: "command_batch_failed",
|
|
3601
|
+
payload: {
|
|
3602
|
+
commandBatchId,
|
|
3603
|
+
error: String(err),
|
|
3604
|
+
partialResults: results
|
|
3605
|
+
},
|
|
3606
|
+
status: "active",
|
|
3607
|
+
currentStepOrder: stepIndexRef.current
|
|
3608
|
+
});
|
|
3609
|
+
}
|
|
3610
|
+
emitIfOpen("tour:action_result", {
|
|
3611
|
+
success: false,
|
|
3612
|
+
reason: "execution_error",
|
|
3613
|
+
error: String(err),
|
|
3614
|
+
results,
|
|
3615
|
+
commandBatchId,
|
|
3616
|
+
runId: runIdRef.current,
|
|
3617
|
+
turnId: turnIdRef.current
|
|
3618
|
+
});
|
|
3619
|
+
clearCommandBatchId();
|
|
3620
|
+
return;
|
|
3621
|
+
}
|
|
3622
|
+
commandInFlightRef.current = false;
|
|
3623
|
+
if (shouldWait && !skipRequestedRef.current) {
|
|
3624
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3625
|
+
const waitCondition = currentStep?.onboarding?.waitCondition;
|
|
3626
|
+
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3627
|
+
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3628
|
+
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3629
|
+
let manualWaitPromise = null;
|
|
3630
|
+
let manualWaitKind = null;
|
|
3631
|
+
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3632
|
+
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3633
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3634
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3635
|
+
if (waitTargetHints) {
|
|
3636
|
+
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3637
|
+
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3638
|
+
manualWaitTarget = preferredWaitTarget;
|
|
3639
|
+
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3631
3640
|
}
|
|
3632
|
-
if (
|
|
3633
|
-
const manualWait = createManualWaitForTarget(
|
|
3641
|
+
if (manualWaitTarget) {
|
|
3642
|
+
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3634
3643
|
manualWaitPromise = manualWait.promise;
|
|
3635
3644
|
manualWaitKind = manualWait.kind;
|
|
3636
3645
|
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3637
|
-
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3638
3646
|
}
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
commandBatchId,
|
|
3658
|
-
results
|
|
3659
|
-
},
|
|
3660
|
-
currentStepOrder: stepIndexRef.current
|
|
3661
|
-
});
|
|
3647
|
+
}
|
|
3648
|
+
if (!manualWaitPromise && preferredWaitTarget) {
|
|
3649
|
+
const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
|
|
3650
|
+
manualWaitPromise = manualWait.promise;
|
|
3651
|
+
manualWaitKind = manualWait.kind;
|
|
3652
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3653
|
+
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3654
|
+
}
|
|
3655
|
+
if (!manualWaitPromise && inputLikeWait) {
|
|
3656
|
+
const firstInput = document.querySelector(
|
|
3657
|
+
'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
|
|
3658
|
+
);
|
|
3659
|
+
if (firstInput) {
|
|
3660
|
+
const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
|
|
3661
|
+
manualWaitPromise = manualWait.promise;
|
|
3662
|
+
manualWaitKind = manualWait.kind;
|
|
3663
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3664
|
+
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3662
3665
|
}
|
|
3663
|
-
socket.emit("tour:action_result", {
|
|
3664
|
-
success: true,
|
|
3665
|
-
waitingForInput: true,
|
|
3666
|
-
results,
|
|
3667
|
-
commandBatchId,
|
|
3668
|
-
runId: runIdRef.current,
|
|
3669
|
-
turnId: turnIdRef.current
|
|
3670
|
-
});
|
|
3671
|
-
clearCommandBatchId();
|
|
3672
|
-
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3673
|
-
if (pendingInputBufRef.current) {
|
|
3674
|
-
const flushed = pendingInputBufRef.current;
|
|
3675
|
-
pendingInputBufRef.current = null;
|
|
3676
|
-
resolve(flushed);
|
|
3677
|
-
return;
|
|
3678
|
-
}
|
|
3679
|
-
voiceInputResolveRef.current = (text) => {
|
|
3680
|
-
voiceInputResolveRef.current = null;
|
|
3681
|
-
resolve(text);
|
|
3682
|
-
};
|
|
3683
|
-
});
|
|
3684
|
-
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3685
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3686
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3687
|
-
voiceInputResolveRef.current = null;
|
|
3688
|
-
setPlaybackState("executing");
|
|
3689
|
-
const transcript = userText.trim();
|
|
3690
|
-
if (!transcript) {
|
|
3691
|
-
return;
|
|
3692
|
-
}
|
|
3693
|
-
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3694
|
-
await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
|
|
3695
|
-
await syncAOM();
|
|
3696
|
-
socket.emit("tour:user_input", {
|
|
3697
|
-
transcript,
|
|
3698
|
-
runId: runIdRef.current,
|
|
3699
|
-
turnId: turnIdRef.current
|
|
3700
|
-
});
|
|
3701
|
-
});
|
|
3702
|
-
return;
|
|
3703
3666
|
}
|
|
3667
|
+
setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
|
|
3704
3668
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3705
3669
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3706
3670
|
stepOrder: stepIndexRef.current,
|
|
3707
|
-
eventType: "
|
|
3671
|
+
eventType: "waiting_for_input",
|
|
3708
3672
|
payload: {
|
|
3709
3673
|
commandBatchId,
|
|
3710
3674
|
results
|
|
@@ -3712,101 +3676,169 @@ function useTourPlayback({
|
|
|
3712
3676
|
currentStepOrder: stepIndexRef.current
|
|
3713
3677
|
});
|
|
3714
3678
|
}
|
|
3715
|
-
|
|
3679
|
+
emitIfOpen("tour:action_result", {
|
|
3716
3680
|
success: true,
|
|
3681
|
+
waitingForInput: true,
|
|
3717
3682
|
results,
|
|
3718
3683
|
commandBatchId,
|
|
3719
3684
|
runId: runIdRef.current,
|
|
3720
3685
|
turnId: turnIdRef.current
|
|
3721
3686
|
});
|
|
3722
3687
|
clearCommandBatchId();
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3688
|
+
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3689
|
+
if (pendingInputBufRef.current) {
|
|
3690
|
+
const flushed = pendingInputBufRef.current;
|
|
3691
|
+
pendingInputBufRef.current = null;
|
|
3692
|
+
resolve(flushed);
|
|
3693
|
+
return;
|
|
3694
|
+
}
|
|
3695
|
+
voiceInputResolveRef.current = (text) => {
|
|
3696
|
+
voiceInputResolveRef.current = null;
|
|
3697
|
+
resolve(text);
|
|
3698
|
+
};
|
|
3699
|
+
});
|
|
3700
|
+
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3701
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3702
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3703
|
+
voiceInputResolveRef.current = null;
|
|
3704
|
+
setPlaybackState("executing");
|
|
3705
|
+
const transcript = userText.trim();
|
|
3706
|
+
if (!transcript) {
|
|
3707
|
+
return;
|
|
3708
|
+
}
|
|
3709
|
+
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3710
|
+
await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
|
|
3711
|
+
await syncAOM();
|
|
3712
|
+
emitIfOpen("tour:user_input", {
|
|
3713
|
+
transcript,
|
|
3714
|
+
runId: runIdRef.current,
|
|
3715
|
+
turnId: turnIdRef.current
|
|
3751
3716
|
});
|
|
3752
|
-
}
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3717
|
+
});
|
|
3718
|
+
return;
|
|
3719
|
+
}
|
|
3720
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3721
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3722
|
+
stepOrder: stepIndexRef.current,
|
|
3723
|
+
eventType: "command_batch_completed",
|
|
3724
|
+
payload: {
|
|
3725
|
+
commandBatchId,
|
|
3726
|
+
results
|
|
3727
|
+
},
|
|
3728
|
+
currentStepOrder: stepIndexRef.current
|
|
3729
|
+
});
|
|
3730
|
+
}
|
|
3731
|
+
emitIfOpen("tour:action_result", {
|
|
3732
|
+
success: true,
|
|
3733
|
+
results,
|
|
3734
|
+
commandBatchId,
|
|
3735
|
+
runId: runIdRef.current,
|
|
3736
|
+
turnId: turnIdRef.current
|
|
3737
|
+
});
|
|
3738
|
+
clearCommandBatchId();
|
|
3739
|
+
});
|
|
3740
|
+
socket.on("tour:start", async (tourData) => {
|
|
3741
|
+
if (isActiveRef.current) return;
|
|
3742
|
+
runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
|
|
3743
|
+
const tour = tourData.tourContext ?? tourRef.current;
|
|
3744
|
+
const expType = experienceTypeRef.current;
|
|
3745
|
+
if (tour?.type && tour.type !== expType) {
|
|
3746
|
+
console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
|
|
3747
|
+
return;
|
|
3748
|
+
}
|
|
3749
|
+
skipRequestedRef.current = false;
|
|
3750
|
+
const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
|
|
3751
|
+
isActiveRef.current = true;
|
|
3752
|
+
setIsActive(true);
|
|
3753
|
+
setActiveTour(tour ?? null);
|
|
3754
|
+
tourRef.current = tour ?? null;
|
|
3755
|
+
setTotalSteps(total);
|
|
3756
|
+
stepIndexRef.current = 0;
|
|
3757
|
+
setCurrentStepIndex(0);
|
|
3758
|
+
setPlaybackState("intro");
|
|
3759
|
+
if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
|
|
3760
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
|
|
3761
|
+
stepOrder: 0,
|
|
3762
|
+
eventType: "tour_started",
|
|
3763
|
+
payload: {
|
|
3764
|
+
totalSteps: total,
|
|
3765
|
+
source: "sdk_test_preview"
|
|
3766
|
+
},
|
|
3767
|
+
currentStepOrder: 0
|
|
3768
|
+
});
|
|
3769
|
+
}
|
|
3770
|
+
try {
|
|
3771
|
+
const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3772
|
+
const aom = generateMinifiedAOM2();
|
|
3773
|
+
if (socketRef.current === socket && socket.connected) {
|
|
3756
3774
|
socket.emit("tour:sync_dom", {
|
|
3757
3775
|
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3758
3776
|
aom: aom.nodes,
|
|
3759
3777
|
domSummary: captureDomSummary()
|
|
3760
3778
|
});
|
|
3761
|
-
} catch (e) {
|
|
3762
|
-
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3763
3779
|
}
|
|
3764
|
-
})
|
|
3765
|
-
|
|
3766
|
-
|
|
3767
|
-
|
|
3768
|
-
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3780
|
+
} catch (e) {
|
|
3781
|
+
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3782
|
+
}
|
|
3783
|
+
});
|
|
3784
|
+
socket.on("tour:update", (payload) => {
|
|
3785
|
+
const updatedTour = payload?.tourContext;
|
|
3786
|
+
if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3789
|
+
tourRef.current = updatedTour;
|
|
3790
|
+
setActiveTour(updatedTour);
|
|
3791
|
+
const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
|
|
3792
|
+
setTotalSteps(nextTotal);
|
|
3793
|
+
if (typeof payload.currentStepIndex === "number") {
|
|
3794
|
+
const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
|
|
3795
|
+
stepIndexRef.current = clampedStepIndex;
|
|
3796
|
+
setCurrentStepIndex(clampedStepIndex);
|
|
3797
|
+
if (nextTotal > 0) {
|
|
3798
|
+
onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
|
|
3781
3799
|
}
|
|
3782
|
-
}
|
|
3783
|
-
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3787
|
-
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3792
|
-
|
|
3793
|
-
|
|
3794
|
-
}
|
|
3795
|
-
}
|
|
3800
|
+
}
|
|
3801
|
+
});
|
|
3802
|
+
socket.on("tour:end", () => {
|
|
3803
|
+
setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
|
|
3804
|
+
handleTourEnd();
|
|
3805
|
+
});
|
|
3806
|
+
socket.on("tour:debug_log", (entry) => {
|
|
3807
|
+
const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
|
|
3808
|
+
if (isDev) {
|
|
3809
|
+
console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
|
|
3810
|
+
if (typeof window !== "undefined") {
|
|
3811
|
+
window.dispatchEvent(new CustomEvent("modelnex-debug", {
|
|
3812
|
+
detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
|
|
3813
|
+
}));
|
|
3796
3814
|
}
|
|
3797
|
-
}
|
|
3798
|
-
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devMode || process.env.NODE_ENV === "development");
|
|
3815
|
+
}
|
|
3799
3816
|
});
|
|
3817
|
+
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3800
3818
|
return () => {
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3819
|
+
const toClose = createdSocket ?? socketRef.current;
|
|
3820
|
+
if (toClose) {
|
|
3821
|
+
toClose.disconnect();
|
|
3804
3822
|
}
|
|
3823
|
+
createdSocket = null;
|
|
3824
|
+
socketRef.current = null;
|
|
3805
3825
|
setServerState(null);
|
|
3806
3826
|
runIdRef.current = null;
|
|
3807
3827
|
turnIdRef.current = null;
|
|
3808
3828
|
};
|
|
3809
|
-
}, [serverUrl, websiteId,
|
|
3829
|
+
}, [serverUrl, websiteId, disabled]);
|
|
3830
|
+
(0, import_react12.useEffect)(() => {
|
|
3831
|
+
if (disabled) return;
|
|
3832
|
+
const s = socketRef.current;
|
|
3833
|
+
const profile = userProfile;
|
|
3834
|
+
if (!s?.connected || !websiteId || !profile?.userId) return;
|
|
3835
|
+
const timer = setTimeout(() => {
|
|
3836
|
+
if (s.connected) {
|
|
3837
|
+
s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3838
|
+
}
|
|
3839
|
+
}, 150);
|
|
3840
|
+
return () => clearTimeout(timer);
|
|
3841
|
+
}, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
|
|
3810
3842
|
(0, import_react12.useEffect)(() => {
|
|
3811
3843
|
if (!showCaptions || !isReviewMode) {
|
|
3812
3844
|
removeCaption();
|
|
@@ -3830,11 +3862,13 @@ function useTourPlayback({
|
|
|
3830
3862
|
if (!socketRef.current?.connected || !isActiveRef.current) return;
|
|
3831
3863
|
const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3832
3864
|
const aom = generateMinifiedAOM2();
|
|
3833
|
-
socketRef.current
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3865
|
+
if (socketRef.current?.connected) {
|
|
3866
|
+
socketRef.current.emit("tour:sync_dom", {
|
|
3867
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3868
|
+
aom: aom.nodes,
|
|
3869
|
+
domSummary: captureDomSummary()
|
|
3870
|
+
});
|
|
3871
|
+
}
|
|
3838
3872
|
}, []);
|
|
3839
3873
|
const interruptExecution = (0, import_react12.useCallback)((transcript) => {
|
|
3840
3874
|
if (!socketRef.current?.connected || !isActiveRef.current) return false;
|
|
@@ -3845,21 +3879,23 @@ function useTourPlayback({
|
|
|
3845
3879
|
removeHighlight();
|
|
3846
3880
|
removeCaption();
|
|
3847
3881
|
voice.stopSpeaking();
|
|
3848
|
-
socketRef.current
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3882
|
+
if (socketRef.current?.connected) {
|
|
3883
|
+
socketRef.current.emit("tour:action_result", {
|
|
3884
|
+
success: true,
|
|
3885
|
+
interrupted: true,
|
|
3886
|
+
transcript,
|
|
3887
|
+
commandBatchId: activeCommandBatchIdRef.current,
|
|
3888
|
+
runId: runIdRef.current,
|
|
3889
|
+
turnId: turnIdRef.current
|
|
3890
|
+
});
|
|
3891
|
+
activeCommandBatchIdRef.current = null;
|
|
3892
|
+
socketRef.current.emit("tour:user_input", {
|
|
3893
|
+
transcript,
|
|
3894
|
+
interrupted: true,
|
|
3895
|
+
runId: runIdRef.current,
|
|
3896
|
+
turnId: turnIdRef.current
|
|
3897
|
+
});
|
|
3898
|
+
}
|
|
3863
3899
|
setPlaybackState("thinking");
|
|
3864
3900
|
return true;
|
|
3865
3901
|
}, [voice]);
|
|
@@ -3977,11 +4013,13 @@ function useTourPlayback({
|
|
|
3977
4013
|
previewRunIdRef.current = null;
|
|
3978
4014
|
}
|
|
3979
4015
|
tourRef.current = tour;
|
|
3980
|
-
socketRef.current
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
4016
|
+
if (socketRef.current?.connected) {
|
|
4017
|
+
socketRef.current.emit("tour:request_start", {
|
|
4018
|
+
tourId: tour.id,
|
|
4019
|
+
previewRunId: previewRunIdRef.current,
|
|
4020
|
+
tourContext: tour
|
|
4021
|
+
});
|
|
4022
|
+
}
|
|
3985
4023
|
}, [serverUrl, websiteId]);
|
|
3986
4024
|
(0, import_react12.useEffect)(() => {
|
|
3987
4025
|
if (!enableAutoDiscovery) return;
|
|
@@ -5429,7 +5467,7 @@ function useVoice(serverUrl) {
|
|
|
5429
5467
|
(async () => {
|
|
5430
5468
|
try {
|
|
5431
5469
|
const ioModule = await import("socket.io-client");
|
|
5432
|
-
const
|
|
5470
|
+
const io3 = ioModule.default || ioModule;
|
|
5433
5471
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
5434
5472
|
audio: {
|
|
5435
5473
|
echoCancellation: true,
|
|
@@ -5451,7 +5489,7 @@ function useVoice(serverUrl) {
|
|
|
5451
5489
|
audioBitsPerSecond: 128e3
|
|
5452
5490
|
});
|
|
5453
5491
|
mediaRecorderRef.current = recorder;
|
|
5454
|
-
const socket =
|
|
5492
|
+
const socket = io3(serverUrl, {
|
|
5455
5493
|
path: "/socket.io",
|
|
5456
5494
|
transports: resolveSocketIoTransports(serverUrl, "websocket-first")
|
|
5457
5495
|
});
|
|
@@ -10972,7 +11010,7 @@ var ModelNexProvider = ({
|
|
|
10972
11010
|
socketId,
|
|
10973
11011
|
devMode
|
|
10974
11012
|
}),
|
|
10975
|
-
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
|
|
11013
|
+
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
|
|
10976
11014
|
);
|
|
10977
11015
|
return import_react21.default.createElement(
|
|
10978
11016
|
ModelNexContext.Provider,
|