@modelnex/sdk 0.5.15 → 0.5.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +570 -565
- package/dist/index.mjs +570 -565
- package/package.json +11 -12
- package/dist/aom-GA6W42DG.mjs +0 -71
- package/dist/aom-J6NYMGDW.mjs +0 -69
- package/dist/chunk-6DZX6EAA.mjs +0 -37
- package/dist/dom-sync-Y7Z7TOU6.mjs +0 -57
package/dist/index.js
CHANGED
|
@@ -2626,6 +2626,7 @@ function isTourEligible(tour, userProfile) {
|
|
|
2626
2626
|
|
|
2627
2627
|
// src/hooks/useTourPlayback.ts
|
|
2628
2628
|
var import_react12 = require("react");
|
|
2629
|
+
var import_socket2 = require("socket.io-client");
|
|
2629
2630
|
|
|
2630
2631
|
// src/utils/retryLookup.ts
|
|
2631
2632
|
var defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -3199,629 +3200,622 @@ function useTourPlayback({
|
|
|
3199
3200
|
(0, import_react12.useEffect)(() => {
|
|
3200
3201
|
if (disabled) return;
|
|
3201
3202
|
if (typeof window === "undefined") return;
|
|
3202
|
-
let cancelled = false;
|
|
3203
3203
|
let createdSocket = null;
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3204
|
+
const socket = (0, import_socket2.io)(serverUrl, {
|
|
3205
|
+
path: "/socket.io",
|
|
3206
|
+
transports: resolveSocketIoTransports(serverUrl, "polling-first"),
|
|
3207
|
+
autoConnect: true,
|
|
3208
|
+
reconnection: true,
|
|
3209
|
+
reconnectionAttempts: 10,
|
|
3210
|
+
reconnectionDelay: 1e3
|
|
3211
|
+
});
|
|
3212
|
+
createdSocket = socket;
|
|
3213
|
+
socketRef.current = socket;
|
|
3214
|
+
socket.on("connect", () => {
|
|
3215
|
+
console.log("[TourClient] Connected to tour agent server:", socket.id);
|
|
3216
|
+
const profile = userProfileRef.current;
|
|
3217
|
+
if (websiteId && profile?.userId && socket.connected) {
|
|
3218
|
+
socket.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3215
3219
|
}
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3220
|
+
});
|
|
3221
|
+
socket.on("tour:server_state", (payload) => {
|
|
3222
|
+
if (typeof payload?.runId === "number") {
|
|
3223
|
+
runIdRef.current = payload.runId;
|
|
3224
|
+
}
|
|
3225
|
+
if (typeof payload?.turnId === "string" || payload?.turnId === null) {
|
|
3226
|
+
turnIdRef.current = payload.turnId ?? null;
|
|
3227
|
+
}
|
|
3228
|
+
setServerState(payload);
|
|
3229
|
+
});
|
|
3230
|
+
socket.on("tour:command_cancel", (payload) => {
|
|
3231
|
+
console.log("[TourClient] Received command_cancel:", payload);
|
|
3232
|
+
if (payload.commandBatchId && activeCommandBatchIdRef.current === payload.commandBatchId) {
|
|
3233
|
+
activeCommandBatchIdRef.current = null;
|
|
3234
|
+
activeExecutionTokenRef.current++;
|
|
3235
|
+
commandInFlightRef.current = false;
|
|
3236
|
+
setPlaybackState("idle");
|
|
3237
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3238
|
+
window.speechSynthesis.cancel();
|
|
3231
3239
|
}
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
if (
|
|
3240
|
+
}
|
|
3241
|
+
});
|
|
3242
|
+
socket.on("tour:command", async (payload) => {
|
|
3243
|
+
const emitIfOpen = (ev, data) => {
|
|
3244
|
+
if (socketRef.current !== socket) return;
|
|
3245
|
+
if (!socket.connected) return;
|
|
3246
|
+
socket.emit(ev, data);
|
|
3247
|
+
};
|
|
3248
|
+
console.log("[TourClient] Received command batch:", payload.stepIndex, payload.commands);
|
|
3249
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3250
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3251
|
+
if (voiceInputResolveRef.current) {
|
|
3252
|
+
const resolvePendingVoiceInput = voiceInputResolveRef.current;
|
|
3253
|
+
voiceInputResolveRef.current = null;
|
|
3254
|
+
resolvePendingVoiceInput("");
|
|
3255
|
+
}
|
|
3256
|
+
setPlaybackState("executing");
|
|
3257
|
+
commandInFlightRef.current = true;
|
|
3258
|
+
const commandBatchId = payload.commandBatchId ?? null;
|
|
3259
|
+
turnIdRef.current = payload.turnId ?? turnIdRef.current;
|
|
3260
|
+
const clearCommandBatchId = () => {
|
|
3261
|
+
if (activeCommandBatchIdRef.current === commandBatchId) {
|
|
3237
3262
|
activeCommandBatchIdRef.current = null;
|
|
3238
|
-
activeExecutionTokenRef.current++;
|
|
3239
|
-
commandInFlightRef.current = false;
|
|
3240
|
-
setPlaybackState("idle");
|
|
3241
|
-
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
3242
|
-
window.speechSynthesis.cancel();
|
|
3243
|
-
}
|
|
3244
3263
|
}
|
|
3245
|
-
}
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
}
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
const
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3264
|
+
};
|
|
3265
|
+
activeCommandBatchIdRef.current = commandBatchId;
|
|
3266
|
+
const executionToken = ++activeExecutionTokenRef.current;
|
|
3267
|
+
const activeTourId = tourRef.current?.id;
|
|
3268
|
+
const activePreviewRunId = previewRunIdRef.current;
|
|
3269
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3270
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3271
|
+
stepOrder: payload.stepIndex,
|
|
3272
|
+
eventType: "command_batch_received",
|
|
3273
|
+
payload: {
|
|
3274
|
+
commandBatchId,
|
|
3275
|
+
commandTypes: Array.isArray(payload.commands) ? payload.commands.map((command) => command?.type || "unknown") : []
|
|
3276
|
+
},
|
|
3277
|
+
currentStepOrder: payload.stepIndex
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
if (typeof payload.stepIndex === "number") {
|
|
3281
|
+
const prevStep = stepIndexRef.current;
|
|
3282
|
+
stepIndexRef.current = payload.stepIndex;
|
|
3283
|
+
setCurrentStepIndex(payload.stepIndex);
|
|
3284
|
+
if (payload.stepIndex !== prevStep) {
|
|
3285
|
+
const tour = tourRef.current;
|
|
3286
|
+
const total = tour?.steps?.length ?? 0;
|
|
3287
|
+
if (tour && total > 0) {
|
|
3288
|
+
onStepChangeRef.current?.(payload.stepIndex, total, tour);
|
|
3267
3289
|
}
|
|
3268
|
-
}
|
|
3269
|
-
|
|
3270
|
-
const executionToken = ++activeExecutionTokenRef.current;
|
|
3271
|
-
const activeTourId = tourRef.current?.id;
|
|
3272
|
-
const activePreviewRunId = previewRunIdRef.current;
|
|
3273
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId && typeof payload.stepIndex === "number") {
|
|
3290
|
+
}
|
|
3291
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3274
3292
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3275
3293
|
stepOrder: payload.stepIndex,
|
|
3276
|
-
eventType: "
|
|
3294
|
+
eventType: "step_started",
|
|
3277
3295
|
payload: {
|
|
3278
|
-
|
|
3279
|
-
|
|
3296
|
+
previousStepOrder: prevStep,
|
|
3297
|
+
stepType: tourRef.current?.steps?.[payload.stepIndex]?.type ?? null
|
|
3280
3298
|
},
|
|
3281
3299
|
currentStepOrder: payload.stepIndex
|
|
3282
3300
|
});
|
|
3283
3301
|
}
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
}
|
|
3307
|
-
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3308
|
-
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3309
|
-
commandInFlightRef.current = false;
|
|
3310
|
-
emitIfOpen("tour:action_result", {
|
|
3311
|
-
success: false,
|
|
3312
|
-
reason: "invalid_commands",
|
|
3313
|
-
commandBatchId,
|
|
3314
|
-
runId: runIdRef.current,
|
|
3315
|
-
turnId: turnIdRef.current
|
|
3316
|
-
});
|
|
3317
|
-
clearCommandBatchId();
|
|
3318
|
-
return;
|
|
3302
|
+
}
|
|
3303
|
+
if (!payload.commands || !Array.isArray(payload.commands)) {
|
|
3304
|
+
console.warn("[TourClient] Payload commands is not an array:", payload);
|
|
3305
|
+
commandInFlightRef.current = false;
|
|
3306
|
+
emitIfOpen("tour:action_result", {
|
|
3307
|
+
success: false,
|
|
3308
|
+
reason: "invalid_commands",
|
|
3309
|
+
commandBatchId,
|
|
3310
|
+
runId: runIdRef.current,
|
|
3311
|
+
turnId: turnIdRef.current
|
|
3312
|
+
});
|
|
3313
|
+
clearCommandBatchId();
|
|
3314
|
+
return;
|
|
3315
|
+
}
|
|
3316
|
+
let shouldWait = false;
|
|
3317
|
+
const results = [];
|
|
3318
|
+
let batchPreferredWaitTarget = null;
|
|
3319
|
+
const assertNotInterrupted = () => {
|
|
3320
|
+
if (executionToken !== activeExecutionTokenRef.current || skipRequestedRef.current) {
|
|
3321
|
+
const error = new Error("interrupted");
|
|
3322
|
+
error.code = "INTERRUPTED";
|
|
3323
|
+
throw error;
|
|
3319
3324
|
}
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
}
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3335
|
-
onRetry: () => {
|
|
3336
|
-
assertNotInterrupted();
|
|
3337
|
-
},
|
|
3338
|
-
resolve: async () => {
|
|
3339
|
-
let targetEl = null;
|
|
3340
|
-
if (params.uid) {
|
|
3341
|
-
const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3342
|
-
targetEl = getElementByUid2(params.uid);
|
|
3343
|
-
}
|
|
3344
|
-
if (!targetEl) {
|
|
3345
|
-
targetEl = resolveElementFromHints({
|
|
3346
|
-
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3347
|
-
testId: params.testId ?? fallbackHints?.testId,
|
|
3348
|
-
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3349
|
-
}, fallbackStep, tagStore);
|
|
3350
|
-
}
|
|
3351
|
-
return targetEl;
|
|
3352
|
-
}
|
|
3353
|
-
});
|
|
3354
|
-
};
|
|
3355
|
-
const executeOne = async (action) => {
|
|
3356
|
-
assertNotInterrupted();
|
|
3357
|
-
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3358
|
-
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3359
|
-
const executeTimeline = async (params = {}) => {
|
|
3360
|
-
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3361
|
-
for (let index = 0; index < segments.length; index += 1) {
|
|
3362
|
-
assertNotInterrupted();
|
|
3363
|
-
const segment = segments[index];
|
|
3364
|
-
const segmentText = (segment?.text ?? "").trim();
|
|
3365
|
-
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3366
|
-
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3367
|
-
if (segmentDelayMs > 0) {
|
|
3368
|
-
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3369
|
-
}
|
|
3370
|
-
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3371
|
-
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3372
|
-
if (!gateTarget) {
|
|
3373
|
-
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3374
|
-
}
|
|
3375
|
-
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3376
|
-
}
|
|
3377
|
-
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3378
|
-
showCaption(segmentText);
|
|
3379
|
-
}
|
|
3380
|
-
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3381
|
-
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3382
|
-
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3383
|
-
onNearEnd: nextSegmentText ? () => {
|
|
3384
|
-
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3385
|
-
} : void 0
|
|
3386
|
-
}) : Promise.resolve();
|
|
3387
|
-
const eventsPromise = (async () => {
|
|
3388
|
-
for (const event of events) {
|
|
3389
|
-
assertNotInterrupted();
|
|
3390
|
-
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3391
|
-
if (delayMs > 0) {
|
|
3392
|
-
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3393
|
-
}
|
|
3394
|
-
if (event?.action) {
|
|
3395
|
-
await executeOne(event.action);
|
|
3396
|
-
}
|
|
3397
|
-
}
|
|
3398
|
-
})();
|
|
3399
|
-
await Promise.all([speechPromise, eventsPromise]);
|
|
3400
|
-
}
|
|
3401
|
-
if (params.removeHighlightAtEnd !== false) {
|
|
3402
|
-
removeHighlight();
|
|
3403
|
-
}
|
|
3404
|
-
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3405
|
-
removeCaption();
|
|
3406
|
-
}
|
|
3407
|
-
return !!params.waitForInput;
|
|
3408
|
-
};
|
|
3409
|
-
if (action.type === "speak") {
|
|
3410
|
-
const text = action.params?.text ?? "";
|
|
3411
|
-
if (!text.trim()) {
|
|
3412
|
-
return { result: null };
|
|
3325
|
+
};
|
|
3326
|
+
const resolveTargetElement2 = async (params = {}, fallbackStep) => {
|
|
3327
|
+
const fallbackHints = fallbackStep?.element ?? null;
|
|
3328
|
+
return retryLookup({
|
|
3329
|
+
timeoutMs: Math.max(0, Number(params.timeoutMs ?? 1800)),
|
|
3330
|
+
pollMs: Math.max(50, Number(params.pollMs ?? 120)),
|
|
3331
|
+
onRetry: () => {
|
|
3332
|
+
assertNotInterrupted();
|
|
3333
|
+
},
|
|
3334
|
+
resolve: async () => {
|
|
3335
|
+
let targetEl = null;
|
|
3336
|
+
if (params.uid) {
|
|
3337
|
+
const { getElementByUid: getElementByUid2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3338
|
+
targetEl = getElementByUid2(params.uid);
|
|
3413
3339
|
}
|
|
3414
|
-
if (
|
|
3415
|
-
|
|
3340
|
+
if (!targetEl) {
|
|
3341
|
+
targetEl = resolveElementFromHints({
|
|
3342
|
+
fingerprint: params.fingerprint ?? fallbackHints?.fingerprint,
|
|
3343
|
+
testId: params.testId ?? fallbackHints?.testId,
|
|
3344
|
+
textContaining: params.textContaining ?? fallbackHints?.textContaining
|
|
3345
|
+
}, fallbackStep, tagStore);
|
|
3416
3346
|
}
|
|
3417
|
-
|
|
3418
|
-
// Schedule narration immediately so the client keeps its speculative,
|
|
3419
|
-
// low-latency behavior, but defer batch completion until playback settles.
|
|
3420
|
-
waitForCompletion: false,
|
|
3421
|
-
interrupt: action.params?.interrupt
|
|
3422
|
-
});
|
|
3423
|
-
return { result: text, settlePromise };
|
|
3424
|
-
}
|
|
3425
|
-
if (action.type === "play_timeline") {
|
|
3426
|
-
const timelineShouldWait = await executeTimeline(action.params);
|
|
3427
|
-
if (timelineShouldWait) shouldWait = true;
|
|
3428
|
-
return { result: "timeline_executed" };
|
|
3347
|
+
return targetEl;
|
|
3429
3348
|
}
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3349
|
+
});
|
|
3350
|
+
};
|
|
3351
|
+
const executeOne = async (action) => {
|
|
3352
|
+
assertNotInterrupted();
|
|
3353
|
+
console.log("[TourClient] Executing action:", action?.type, action?.params ? JSON.stringify(action.params).slice(0, 120) : "");
|
|
3354
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3355
|
+
const executeTimeline = async (params = {}) => {
|
|
3356
|
+
const segments = Array.isArray(params.segments) ? params.segments : [];
|
|
3357
|
+
for (let index = 0; index < segments.length; index += 1) {
|
|
3358
|
+
assertNotInterrupted();
|
|
3359
|
+
const segment = segments[index];
|
|
3360
|
+
const segmentText = (segment?.text ?? "").trim();
|
|
3361
|
+
const segmentDelayMs = Math.max(0, Number(segment?.delayMs ?? 0));
|
|
3362
|
+
const events = Array.isArray(segment?.events) ? segment.events : [];
|
|
3363
|
+
if (segmentDelayMs > 0) {
|
|
3364
|
+
await new Promise((resolve) => setTimeout(resolve, segmentDelayMs));
|
|
3365
|
+
}
|
|
3366
|
+
if (segment?.gate?.type === "user_action" && segment.gate.target && segment.gate.event) {
|
|
3367
|
+
const gateTarget = await resolveTargetElement2(segment.gate.target, currentStep);
|
|
3368
|
+
if (!gateTarget) {
|
|
3369
|
+
throw new Error(`timeline gate target not found for ${segment.gate.event}`);
|
|
3435
3370
|
}
|
|
3436
|
-
|
|
3437
|
-
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3438
|
-
return { result: "highlighted" };
|
|
3371
|
+
await waitForUserAction(gateTarget, segment.gate.event);
|
|
3439
3372
|
}
|
|
3440
|
-
if (
|
|
3441
|
-
|
|
3442
|
-
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3443
|
-
);
|
|
3373
|
+
if (segmentText && showCaptionsRef.current && reviewModeRef.current) {
|
|
3374
|
+
showCaption(segmentText);
|
|
3444
3375
|
}
|
|
3445
|
-
|
|
3376
|
+
const nextSegmentText = (segments[index + 1]?.text ?? "").trim();
|
|
3377
|
+
const speechPromise = segmentText ? voice.speak(segmentText, tourRef.current?.voice?.ttsVoice, {
|
|
3378
|
+
prefetchLeadMs: tourRef.current?.voice?.ttsPrefetchLeadMs ?? currentStep?.execution?.ttsPrefetchLeadMs ?? 2e3,
|
|
3379
|
+
onNearEnd: nextSegmentText ? () => {
|
|
3380
|
+
void voice.prefetchSpeech(nextSegmentText, tourRef.current?.voice?.ttsVoice);
|
|
3381
|
+
} : void 0
|
|
3382
|
+
}) : Promise.resolve();
|
|
3383
|
+
const eventsPromise = (async () => {
|
|
3384
|
+
for (const event of events) {
|
|
3385
|
+
assertNotInterrupted();
|
|
3386
|
+
const delayMs = Math.max(0, Number(event?.delayMs ?? 0));
|
|
3387
|
+
if (delayMs > 0) {
|
|
3388
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
3389
|
+
}
|
|
3390
|
+
if (event?.action) {
|
|
3391
|
+
await executeOne(event.action);
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
})();
|
|
3395
|
+
await Promise.all([speechPromise, eventsPromise]);
|
|
3446
3396
|
}
|
|
3447
|
-
if (
|
|
3397
|
+
if (params.removeHighlightAtEnd !== false) {
|
|
3448
3398
|
removeHighlight();
|
|
3449
|
-
return { result: "highlight_removed" };
|
|
3450
3399
|
}
|
|
3451
|
-
if (
|
|
3452
|
-
|
|
3453
|
-
if (!targetEl) {
|
|
3454
|
-
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3455
|
-
throw new Error(
|
|
3456
|
-
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3457
|
-
);
|
|
3458
|
-
}
|
|
3459
|
-
removeHighlight();
|
|
3460
|
-
await performInteractiveClick(targetEl);
|
|
3461
|
-
const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3462
|
-
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3463
|
-
return { result: "clicked" };
|
|
3400
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3401
|
+
removeCaption();
|
|
3464
3402
|
}
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
}
|
|
3472
|
-
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3473
|
-
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3474
|
-
await performInteractiveFill(targetEl, value);
|
|
3475
|
-
return { result: value };
|
|
3403
|
+
return !!params.waitForInput;
|
|
3404
|
+
};
|
|
3405
|
+
if (action.type === "speak") {
|
|
3406
|
+
const text = action.params?.text ?? "";
|
|
3407
|
+
if (!text.trim()) {
|
|
3408
|
+
return { result: null };
|
|
3476
3409
|
}
|
|
3477
|
-
if (
|
|
3478
|
-
|
|
3479
|
-
const html2canvas2 = html2canvasModule.default;
|
|
3480
|
-
const canvas = await html2canvas2(document.body, {
|
|
3481
|
-
useCORS: true,
|
|
3482
|
-
allowTaint: true,
|
|
3483
|
-
scale: Math.min(window.devicePixelRatio, 2),
|
|
3484
|
-
width: window.innerWidth,
|
|
3485
|
-
height: window.innerHeight,
|
|
3486
|
-
x: window.scrollX,
|
|
3487
|
-
y: window.scrollY,
|
|
3488
|
-
logging: false
|
|
3489
|
-
});
|
|
3490
|
-
return { result: canvas.toDataURL("image/png") };
|
|
3410
|
+
if (showCaptionsRef.current && reviewModeRef.current) {
|
|
3411
|
+
showCaption(text);
|
|
3491
3412
|
}
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3413
|
+
const settlePromise = voice.speak(text, tourRef.current?.voice?.ttsVoice, {
|
|
3414
|
+
// Schedule narration immediately so the client keeps its speculative,
|
|
3415
|
+
// low-latency behavior, but defer batch completion until playback settles.
|
|
3416
|
+
waitForCompletion: false,
|
|
3417
|
+
interrupt: action.params?.interrupt
|
|
3418
|
+
});
|
|
3419
|
+
return { result: text, settlePromise };
|
|
3420
|
+
}
|
|
3421
|
+
if (action.type === "play_timeline") {
|
|
3422
|
+
const timelineShouldWait = await executeTimeline(action.params);
|
|
3423
|
+
if (timelineShouldWait) shouldWait = true;
|
|
3424
|
+
return { result: "timeline_executed" };
|
|
3425
|
+
}
|
|
3426
|
+
if (action.type === "highlight_element") {
|
|
3427
|
+
const resolvedEl = await resolveTargetElement2(action.params, currentStep);
|
|
3428
|
+
if (resolvedEl) {
|
|
3429
|
+
if (isEditableWaitTarget(resolvedEl)) {
|
|
3430
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(resolvedEl);
|
|
3496
3431
|
}
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
return { result: nextUrl };
|
|
3432
|
+
showHighlight(resolvedEl, action.params?.label || action.label);
|
|
3433
|
+
resolvedEl.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
3434
|
+
return { result: "highlighted" };
|
|
3501
3435
|
}
|
|
3502
|
-
if (action.
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
}
|
|
3507
|
-
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3508
|
-
const res = await fetch(url, {
|
|
3509
|
-
method: "POST",
|
|
3510
|
-
headers: { "Content-Type": "application/json" },
|
|
3511
|
-
body: JSON.stringify({
|
|
3512
|
-
command: action.params?.command,
|
|
3513
|
-
socketId: agentSocketId
|
|
3514
|
-
})
|
|
3515
|
-
});
|
|
3516
|
-
if (!res.ok) {
|
|
3517
|
-
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3518
|
-
}
|
|
3519
|
-
if (action.params?.wait !== false) {
|
|
3520
|
-
await res.json();
|
|
3521
|
-
}
|
|
3522
|
-
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3523
|
-
await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3524
|
-
await syncAOM();
|
|
3525
|
-
return { result: action.params?.command ?? "executed" };
|
|
3436
|
+
if (!action.params?.optional) {
|
|
3437
|
+
throw new Error(
|
|
3438
|
+
`highlight_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3439
|
+
);
|
|
3526
3440
|
}
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3441
|
+
return { result: "highlight_optional_miss" };
|
|
3442
|
+
}
|
|
3443
|
+
if (action.type === "remove_highlight") {
|
|
3444
|
+
removeHighlight();
|
|
3445
|
+
return { result: "highlight_removed" };
|
|
3446
|
+
}
|
|
3447
|
+
if (action.type === "click_element") {
|
|
3448
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3449
|
+
if (!targetEl) {
|
|
3450
|
+
if (action.params?.optional) return { result: "click_optional_miss" };
|
|
3451
|
+
throw new Error(
|
|
3452
|
+
`click_element target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3453
|
+
);
|
|
3535
3454
|
}
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
|
|
3455
|
+
removeHighlight();
|
|
3456
|
+
await performInteractiveClick(targetEl);
|
|
3457
|
+
const { waitForDomSettle: waitForDomSettleClick } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3458
|
+
await waitForDomSettleClick({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3459
|
+
return { result: "clicked" };
|
|
3460
|
+
}
|
|
3461
|
+
if (action.type === "fill_input") {
|
|
3462
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3463
|
+
if (!targetEl) {
|
|
3464
|
+
throw new Error(
|
|
3465
|
+
`fill_input target not found (${action.params?.uid || action.params?.testId || action.params?.fingerprint || action.params?.textContaining || currentStep?.element?.testId || currentStep?.element?.fingerprint || currentStep?.element?.textContaining || "unknown target"})`
|
|
3466
|
+
);
|
|
3539
3467
|
}
|
|
3540
|
-
|
|
3541
|
-
|
|
3542
|
-
|
|
3468
|
+
const value = typeof action.params?.value === "string" ? action.params.value : "";
|
|
3469
|
+
batchPreferredWaitTarget = resolveWaitTargetElement(targetEl);
|
|
3470
|
+
await performInteractiveFill(targetEl, value);
|
|
3471
|
+
return { result: value };
|
|
3472
|
+
}
|
|
3473
|
+
if (action.type === "take_screenshot") {
|
|
3474
|
+
const html2canvasModule = await import("html2canvas");
|
|
3475
|
+
const html2canvas2 = html2canvasModule.default;
|
|
3476
|
+
const canvas = await html2canvas2(document.body, {
|
|
3477
|
+
useCORS: true,
|
|
3478
|
+
allowTaint: true,
|
|
3479
|
+
scale: Math.min(window.devicePixelRatio, 2),
|
|
3480
|
+
width: window.innerWidth,
|
|
3481
|
+
height: window.innerHeight,
|
|
3482
|
+
x: window.scrollX,
|
|
3483
|
+
y: window.scrollY,
|
|
3484
|
+
logging: false
|
|
3485
|
+
});
|
|
3486
|
+
return { result: canvas.toDataURL("image/png") };
|
|
3487
|
+
}
|
|
3488
|
+
if (action.type === "navigate_to_url") {
|
|
3489
|
+
const nextUrl = typeof action.params?.url === "string" ? action.params.url : "";
|
|
3490
|
+
if (!nextUrl) {
|
|
3491
|
+
throw new Error("navigate_to_url missing url");
|
|
3543
3492
|
}
|
|
3544
|
-
|
|
3545
|
-
|
|
3546
|
-
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
const isTerminal = isTerminalAction(command);
|
|
3554
|
-
if (isTerminal) {
|
|
3555
|
-
await Promise.all(pendingUIActions);
|
|
3556
|
-
pendingUIActions.length = 0;
|
|
3557
|
-
}
|
|
3558
|
-
const executionPromise = (async () => {
|
|
3559
|
-
const execution = await executeOne(command);
|
|
3560
|
-
await execution.settlePromise;
|
|
3561
|
-
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3562
|
-
})();
|
|
3563
|
-
if (isTerminal) {
|
|
3564
|
-
await executionPromise;
|
|
3565
|
-
} else {
|
|
3566
|
-
pendingUIActions.push(executionPromise);
|
|
3567
|
-
}
|
|
3493
|
+
await navigateToTourUrl(nextUrl);
|
|
3494
|
+
const { waitForDomSettle: waitForDomSettleNav } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3495
|
+
await waitForDomSettleNav({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3496
|
+
return { result: nextUrl };
|
|
3497
|
+
}
|
|
3498
|
+
if (action.type === "execute_agent_action") {
|
|
3499
|
+
const agentSocketId = socketIdRef.current ?? socketRef.current?.id;
|
|
3500
|
+
if (!agentSocketId) {
|
|
3501
|
+
throw new Error("No socketId available for execute_agent_action");
|
|
3568
3502
|
}
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3503
|
+
const url = getAgentCommandUrl(serverUrl, commandUrlRef.current);
|
|
3504
|
+
const res = await fetch(url, {
|
|
3505
|
+
method: "POST",
|
|
3506
|
+
headers: { "Content-Type": "application/json" },
|
|
3507
|
+
body: JSON.stringify({
|
|
3508
|
+
command: action.params?.command,
|
|
3509
|
+
socketId: agentSocketId
|
|
3510
|
+
})
|
|
3572
3511
|
});
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
commandInFlightRef.current = false;
|
|
3576
|
-
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3577
|
-
if (interrupted) {
|
|
3578
|
-
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3579
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3580
|
-
stepOrder: stepIndexRef.current,
|
|
3581
|
-
eventType: "command_batch_interrupted",
|
|
3582
|
-
payload: {
|
|
3583
|
-
commandBatchId,
|
|
3584
|
-
partialResults: results
|
|
3585
|
-
},
|
|
3586
|
-
currentStepOrder: stepIndexRef.current
|
|
3587
|
-
});
|
|
3588
|
-
}
|
|
3589
|
-
emitIfOpen("tour:action_result", {
|
|
3590
|
-
success: true,
|
|
3591
|
-
interrupted: true,
|
|
3592
|
-
results,
|
|
3593
|
-
commandBatchId,
|
|
3594
|
-
runId: runIdRef.current,
|
|
3595
|
-
turnId: turnIdRef.current
|
|
3596
|
-
});
|
|
3597
|
-
clearCommandBatchId();
|
|
3598
|
-
return;
|
|
3512
|
+
if (!res.ok) {
|
|
3513
|
+
throw new Error(`execute_agent_action failed: ${await res.text()}`);
|
|
3599
3514
|
}
|
|
3600
|
-
|
|
3601
|
-
|
|
3602
|
-
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3603
|
-
stepOrder: stepIndexRef.current,
|
|
3604
|
-
eventType: "command_batch_failed",
|
|
3605
|
-
payload: {
|
|
3606
|
-
commandBatchId,
|
|
3607
|
-
error: String(err),
|
|
3608
|
-
partialResults: results
|
|
3609
|
-
},
|
|
3610
|
-
status: "active",
|
|
3611
|
-
currentStepOrder: stepIndexRef.current
|
|
3612
|
-
});
|
|
3515
|
+
if (action.params?.wait !== false) {
|
|
3516
|
+
await res.json();
|
|
3613
3517
|
}
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
results,
|
|
3619
|
-
commandBatchId,
|
|
3620
|
-
runId: runIdRef.current,
|
|
3621
|
-
turnId: turnIdRef.current
|
|
3622
|
-
});
|
|
3623
|
-
clearCommandBatchId();
|
|
3624
|
-
return;
|
|
3518
|
+
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3519
|
+
await waitForDomSettle2({ timeoutMs: 3e3, debounceMs: 300 });
|
|
3520
|
+
await syncAOM();
|
|
3521
|
+
return { result: action.params?.command ?? "executed" };
|
|
3625
3522
|
}
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3631
|
-
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3632
|
-
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3633
|
-
let manualWaitPromise = null;
|
|
3634
|
-
let manualWaitKind = null;
|
|
3635
|
-
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3636
|
-
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3637
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3638
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3639
|
-
if (waitTargetHints) {
|
|
3640
|
-
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3641
|
-
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3642
|
-
manualWaitTarget = preferredWaitTarget;
|
|
3643
|
-
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3644
|
-
}
|
|
3645
|
-
if (manualWaitTarget) {
|
|
3646
|
-
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3647
|
-
manualWaitPromise = manualWait.promise;
|
|
3648
|
-
manualWaitKind = manualWait.kind;
|
|
3649
|
-
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3650
|
-
}
|
|
3523
|
+
if (action.type === "wait_for_user_action") {
|
|
3524
|
+
const targetEl = await resolveTargetElement2(action.params, currentStep);
|
|
3525
|
+
if (!targetEl) {
|
|
3526
|
+
throw new Error("wait_for_user_action target not found");
|
|
3651
3527
|
}
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
3655
|
-
|
|
3656
|
-
|
|
3657
|
-
|
|
3528
|
+
const eventName = action.params?.event ?? "click";
|
|
3529
|
+
await waitForUserAction(targetEl, eventName);
|
|
3530
|
+
return { result: `waited_for_${eventName}` };
|
|
3531
|
+
}
|
|
3532
|
+
if (action.type === "wait_for_input") {
|
|
3533
|
+
shouldWait = true;
|
|
3534
|
+
return { result: "waiting_for_input" };
|
|
3535
|
+
}
|
|
3536
|
+
if (action.type === "end_tour") {
|
|
3537
|
+
handleTourEnd();
|
|
3538
|
+
return { result: "ended" };
|
|
3539
|
+
}
|
|
3540
|
+
console.warn("[TourClient] Unknown action type:", action?.type, "- skipping");
|
|
3541
|
+
return { result: "unknown_action_skipped" };
|
|
3542
|
+
};
|
|
3543
|
+
try {
|
|
3544
|
+
const resultsBuffer = new Array(payload.commands.length);
|
|
3545
|
+
const pendingUIActions = [];
|
|
3546
|
+
for (let commandIndex = 0; commandIndex < payload.commands.length; commandIndex += 1) {
|
|
3547
|
+
const command = payload.commands[commandIndex];
|
|
3548
|
+
assertNotInterrupted();
|
|
3549
|
+
const isTerminal = isTerminalAction(command);
|
|
3550
|
+
if (isTerminal) {
|
|
3551
|
+
await Promise.all(pendingUIActions);
|
|
3552
|
+
pendingUIActions.length = 0;
|
|
3658
3553
|
}
|
|
3659
|
-
|
|
3660
|
-
const
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3669
|
-
}
|
|
3554
|
+
const executionPromise = (async () => {
|
|
3555
|
+
const execution = await executeOne(command);
|
|
3556
|
+
await execution.settlePromise;
|
|
3557
|
+
resultsBuffer[commandIndex] = { type: command.type, success: true, result: execution.result };
|
|
3558
|
+
})();
|
|
3559
|
+
if (isTerminal) {
|
|
3560
|
+
await executionPromise;
|
|
3561
|
+
} else {
|
|
3562
|
+
pendingUIActions.push(executionPromise);
|
|
3670
3563
|
}
|
|
3671
|
-
|
|
3564
|
+
}
|
|
3565
|
+
await Promise.all(pendingUIActions);
|
|
3566
|
+
resultsBuffer.forEach((res) => {
|
|
3567
|
+
if (res) results.push(res);
|
|
3568
|
+
});
|
|
3569
|
+
await syncAOM();
|
|
3570
|
+
} catch (err) {
|
|
3571
|
+
commandInFlightRef.current = false;
|
|
3572
|
+
const interrupted = err?.code === "INTERRUPTED" || String(err) === "Error: interrupted";
|
|
3573
|
+
if (interrupted) {
|
|
3672
3574
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3673
3575
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3674
3576
|
stepOrder: stepIndexRef.current,
|
|
3675
|
-
eventType: "
|
|
3577
|
+
eventType: "command_batch_interrupted",
|
|
3676
3578
|
payload: {
|
|
3677
3579
|
commandBatchId,
|
|
3678
|
-
results
|
|
3580
|
+
partialResults: results
|
|
3679
3581
|
},
|
|
3680
3582
|
currentStepOrder: stepIndexRef.current
|
|
3681
3583
|
});
|
|
3682
3584
|
}
|
|
3683
3585
|
emitIfOpen("tour:action_result", {
|
|
3684
3586
|
success: true,
|
|
3685
|
-
|
|
3587
|
+
interrupted: true,
|
|
3686
3588
|
results,
|
|
3687
3589
|
commandBatchId,
|
|
3688
3590
|
runId: runIdRef.current,
|
|
3689
3591
|
turnId: turnIdRef.current
|
|
3690
3592
|
});
|
|
3691
3593
|
clearCommandBatchId();
|
|
3692
|
-
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3693
|
-
if (pendingInputBufRef.current) {
|
|
3694
|
-
const flushed = pendingInputBufRef.current;
|
|
3695
|
-
pendingInputBufRef.current = null;
|
|
3696
|
-
resolve(flushed);
|
|
3697
|
-
return;
|
|
3698
|
-
}
|
|
3699
|
-
voiceInputResolveRef.current = (text) => {
|
|
3700
|
-
voiceInputResolveRef.current = null;
|
|
3701
|
-
resolve(text);
|
|
3702
|
-
};
|
|
3703
|
-
});
|
|
3704
|
-
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3705
|
-
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3706
|
-
pendingManualWaitCleanupRef.current = null;
|
|
3707
|
-
voiceInputResolveRef.current = null;
|
|
3708
|
-
setPlaybackState("executing");
|
|
3709
|
-
const transcript = userText.trim();
|
|
3710
|
-
if (!transcript) {
|
|
3711
|
-
return;
|
|
3712
|
-
}
|
|
3713
|
-
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3714
|
-
await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
|
|
3715
|
-
await syncAOM();
|
|
3716
|
-
emitIfOpen("tour:user_input", {
|
|
3717
|
-
transcript,
|
|
3718
|
-
runId: runIdRef.current,
|
|
3719
|
-
turnId: turnIdRef.current
|
|
3720
|
-
});
|
|
3721
|
-
});
|
|
3722
3594
|
return;
|
|
3723
3595
|
}
|
|
3596
|
+
console.error("[TourClient] Command batch execution failed:", err);
|
|
3724
3597
|
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3725
3598
|
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3726
3599
|
stepOrder: stepIndexRef.current,
|
|
3727
|
-
eventType: "
|
|
3600
|
+
eventType: "command_batch_failed",
|
|
3728
3601
|
payload: {
|
|
3729
3602
|
commandBatchId,
|
|
3730
|
-
|
|
3603
|
+
error: String(err),
|
|
3604
|
+
partialResults: results
|
|
3731
3605
|
},
|
|
3606
|
+
status: "active",
|
|
3732
3607
|
currentStepOrder: stepIndexRef.current
|
|
3733
3608
|
});
|
|
3734
3609
|
}
|
|
3735
3610
|
emitIfOpen("tour:action_result", {
|
|
3736
|
-
success:
|
|
3611
|
+
success: false,
|
|
3612
|
+
reason: "execution_error",
|
|
3613
|
+
error: String(err),
|
|
3737
3614
|
results,
|
|
3738
3615
|
commandBatchId,
|
|
3739
3616
|
runId: runIdRef.current,
|
|
3740
3617
|
turnId: turnIdRef.current
|
|
3741
3618
|
});
|
|
3742
3619
|
clearCommandBatchId();
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
const
|
|
3748
|
-
const
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
3620
|
+
return;
|
|
3621
|
+
}
|
|
3622
|
+
commandInFlightRef.current = false;
|
|
3623
|
+
if (shouldWait && !skipRequestedRef.current) {
|
|
3624
|
+
const currentStep = tourRef.current?.steps?.[stepIndexRef.current] ?? null;
|
|
3625
|
+
const waitCondition = currentStep?.onboarding?.waitCondition;
|
|
3626
|
+
const waitTargetHints = waitCondition?.target ?? currentStep?.onboarding?.waitTarget ?? currentStep?.element;
|
|
3627
|
+
const waitEvent = waitCondition?.event ?? currentStep?.onboarding?.expectedUserAction ?? "input";
|
|
3628
|
+
const inputLikeWait = isInputLikeWait(waitEvent, currentStep);
|
|
3629
|
+
let manualWaitPromise = null;
|
|
3630
|
+
let manualWaitKind = null;
|
|
3631
|
+
const highlightedWaitTarget = lastHighlightTarget ? resolveWaitTargetElement(lastHighlightTarget) : null;
|
|
3632
|
+
const preferredWaitTarget = inputLikeWait ? batchPreferredWaitTarget ?? highlightedWaitTarget : highlightedWaitTarget;
|
|
3633
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3634
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3635
|
+
if (waitTargetHints) {
|
|
3636
|
+
let manualWaitTarget = await resolveTargetElement2(waitTargetHints, currentStep);
|
|
3637
|
+
if (inputLikeWait && preferredWaitTarget && manualWaitTarget && manualWaitTarget !== preferredWaitTarget && !isEditableWaitTarget(manualWaitTarget) && isEditableWaitTarget(preferredWaitTarget)) {
|
|
3638
|
+
manualWaitTarget = preferredWaitTarget;
|
|
3639
|
+
console.log("[TourClient] wait_for_input: preferring current editable target over hinted step target", manualWaitTarget);
|
|
3640
|
+
}
|
|
3641
|
+
if (manualWaitTarget) {
|
|
3642
|
+
const manualWait = createManualWaitForTarget(manualWaitTarget, waitEvent, currentStep);
|
|
3643
|
+
manualWaitPromise = manualWait.promise;
|
|
3644
|
+
manualWaitKind = manualWait.kind;
|
|
3645
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3646
|
+
}
|
|
3752
3647
|
}
|
|
3753
|
-
|
|
3754
|
-
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
|
|
3760
|
-
|
|
3761
|
-
|
|
3762
|
-
|
|
3763
|
-
|
|
3764
|
-
|
|
3765
|
-
|
|
3766
|
-
|
|
3648
|
+
if (!manualWaitPromise && preferredWaitTarget) {
|
|
3649
|
+
const manualWait = createManualWaitForTarget(preferredWaitTarget, waitEvent, currentStep);
|
|
3650
|
+
manualWaitPromise = manualWait.promise;
|
|
3651
|
+
manualWaitKind = manualWait.kind;
|
|
3652
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3653
|
+
console.log("[TourClient] wait_for_input: using current editable target as fallback wait target", preferredWaitTarget);
|
|
3654
|
+
}
|
|
3655
|
+
if (!manualWaitPromise && inputLikeWait) {
|
|
3656
|
+
const firstInput = document.querySelector(
|
|
3657
|
+
'input:not([type="hidden"]):not([disabled]), textarea:not([disabled]), [contenteditable="true"], [role="textbox"]'
|
|
3658
|
+
);
|
|
3659
|
+
if (firstInput) {
|
|
3660
|
+
const manualWait = createManualWaitForTarget(firstInput, waitEvent, currentStep);
|
|
3661
|
+
manualWaitPromise = manualWait.promise;
|
|
3662
|
+
manualWaitKind = manualWait.kind;
|
|
3663
|
+
pendingManualWaitCleanupRef.current = manualWait.cleanup;
|
|
3664
|
+
console.log("[TourClient] wait_for_input: no target found, falling back to first visible editable element", firstInput);
|
|
3665
|
+
}
|
|
3666
|
+
}
|
|
3667
|
+
setPlaybackState(manualWaitKind ? "waiting_input" : "waiting_voice");
|
|
3668
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3669
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3670
|
+
stepOrder: stepIndexRef.current,
|
|
3671
|
+
eventType: "waiting_for_input",
|
|
3767
3672
|
payload: {
|
|
3768
|
-
|
|
3769
|
-
|
|
3673
|
+
commandBatchId,
|
|
3674
|
+
results
|
|
3770
3675
|
},
|
|
3771
|
-
currentStepOrder:
|
|
3676
|
+
currentStepOrder: stepIndexRef.current
|
|
3772
3677
|
});
|
|
3773
3678
|
}
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
3782
|
-
|
|
3679
|
+
emitIfOpen("tour:action_result", {
|
|
3680
|
+
success: true,
|
|
3681
|
+
waitingForInput: true,
|
|
3682
|
+
results,
|
|
3683
|
+
commandBatchId,
|
|
3684
|
+
runId: runIdRef.current,
|
|
3685
|
+
turnId: turnIdRef.current
|
|
3686
|
+
});
|
|
3687
|
+
clearCommandBatchId();
|
|
3688
|
+
const voiceOrTextWaitPromise = new Promise((resolve) => {
|
|
3689
|
+
if (pendingInputBufRef.current) {
|
|
3690
|
+
const flushed = pendingInputBufRef.current;
|
|
3691
|
+
pendingInputBufRef.current = null;
|
|
3692
|
+
resolve(flushed);
|
|
3693
|
+
return;
|
|
3783
3694
|
}
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
3695
|
+
voiceInputResolveRef.current = (text) => {
|
|
3696
|
+
voiceInputResolveRef.current = null;
|
|
3697
|
+
resolve(text);
|
|
3698
|
+
};
|
|
3699
|
+
});
|
|
3700
|
+
Promise.race([voiceOrTextWaitPromise, manualWaitPromise].filter(Boolean)).then(async (userText) => {
|
|
3701
|
+
runCleanup(pendingManualWaitCleanupRef.current);
|
|
3702
|
+
pendingManualWaitCleanupRef.current = null;
|
|
3703
|
+
voiceInputResolveRef.current = null;
|
|
3704
|
+
setPlaybackState("executing");
|
|
3705
|
+
const transcript = userText.trim();
|
|
3706
|
+
if (!transcript) {
|
|
3707
|
+
return;
|
|
3708
|
+
}
|
|
3709
|
+
const { waitForDomSettle: waitForDomSettle2 } = await Promise.resolve().then(() => (init_dom_sync(), dom_sync_exports));
|
|
3710
|
+
await waitForDomSettle2({ timeoutMs: 1500, debounceMs: 200 });
|
|
3711
|
+
await syncAOM();
|
|
3712
|
+
emitIfOpen("tour:user_input", {
|
|
3713
|
+
transcript,
|
|
3714
|
+
runId: runIdRef.current,
|
|
3715
|
+
turnId: turnIdRef.current
|
|
3716
|
+
});
|
|
3717
|
+
});
|
|
3718
|
+
return;
|
|
3719
|
+
}
|
|
3720
|
+
if (reviewModeRef.current && activeTourId && activePreviewRunId) {
|
|
3721
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, activeTourId, activePreviewRunId, websiteId, {
|
|
3722
|
+
stepOrder: stepIndexRef.current,
|
|
3723
|
+
eventType: "command_batch_completed",
|
|
3724
|
+
payload: {
|
|
3725
|
+
commandBatchId,
|
|
3726
|
+
results
|
|
3727
|
+
},
|
|
3728
|
+
currentStepOrder: stepIndexRef.current
|
|
3729
|
+
});
|
|
3730
|
+
}
|
|
3731
|
+
emitIfOpen("tour:action_result", {
|
|
3732
|
+
success: true,
|
|
3733
|
+
results,
|
|
3734
|
+
commandBatchId,
|
|
3735
|
+
runId: runIdRef.current,
|
|
3736
|
+
turnId: turnIdRef.current
|
|
3787
3737
|
});
|
|
3788
|
-
|
|
3789
|
-
|
|
3790
|
-
|
|
3791
|
-
|
|
3738
|
+
clearCommandBatchId();
|
|
3739
|
+
});
|
|
3740
|
+
socket.on("tour:start", async (tourData) => {
|
|
3741
|
+
if (isActiveRef.current) return;
|
|
3742
|
+
runIdRef.current = typeof tourData.runId === "number" ? tourData.runId : runIdRef.current;
|
|
3743
|
+
const tour = tourData.tourContext ?? tourRef.current;
|
|
3744
|
+
const expType = experienceTypeRef.current;
|
|
3745
|
+
if (tour?.type && tour.type !== expType) {
|
|
3746
|
+
console.log(`[TourClient] Ignoring ${tour.type} start (this hook is for ${expType})`);
|
|
3747
|
+
return;
|
|
3748
|
+
}
|
|
3749
|
+
skipRequestedRef.current = false;
|
|
3750
|
+
const total = tourData.totalSteps ?? tour?.steps?.length ?? 0;
|
|
3751
|
+
isActiveRef.current = true;
|
|
3752
|
+
setIsActive(true);
|
|
3753
|
+
setActiveTour(tour ?? null);
|
|
3754
|
+
tourRef.current = tour ?? null;
|
|
3755
|
+
setTotalSteps(total);
|
|
3756
|
+
stepIndexRef.current = 0;
|
|
3757
|
+
setCurrentStepIndex(0);
|
|
3758
|
+
setPlaybackState("intro");
|
|
3759
|
+
if (reviewModeRef.current && tour?.id && previewRunIdRef.current) {
|
|
3760
|
+
void logPreviewEvent(serverUrl, toursApiBaseRef.current, tour.id, previewRunIdRef.current, websiteId, {
|
|
3761
|
+
stepOrder: 0,
|
|
3762
|
+
eventType: "tour_started",
|
|
3763
|
+
payload: {
|
|
3764
|
+
totalSteps: total,
|
|
3765
|
+
source: "sdk_test_preview"
|
|
3766
|
+
},
|
|
3767
|
+
currentStepOrder: 0
|
|
3768
|
+
});
|
|
3769
|
+
}
|
|
3770
|
+
try {
|
|
3771
|
+
const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3772
|
+
const aom = generateMinifiedAOM2();
|
|
3773
|
+
if (socketRef.current === socket && socket.connected) {
|
|
3774
|
+
socket.emit("tour:sync_dom", {
|
|
3775
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3776
|
+
aom: aom.nodes,
|
|
3777
|
+
domSummary: captureDomSummary()
|
|
3778
|
+
});
|
|
3792
3779
|
}
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3780
|
+
} catch (e) {
|
|
3781
|
+
console.warn("[TourClient] Initial DOM sync failed:", e);
|
|
3782
|
+
}
|
|
3783
|
+
});
|
|
3784
|
+
socket.on("tour:update", (payload) => {
|
|
3785
|
+
const updatedTour = payload?.tourContext;
|
|
3786
|
+
if (!updatedTour?.id || updatedTour.id !== tourRef.current?.id) {
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3789
|
+
tourRef.current = updatedTour;
|
|
3790
|
+
setActiveTour(updatedTour);
|
|
3791
|
+
const nextTotal = payload.totalSteps ?? updatedTour.steps?.length ?? 0;
|
|
3792
|
+
setTotalSteps(nextTotal);
|
|
3793
|
+
if (typeof payload.currentStepIndex === "number") {
|
|
3794
|
+
const clampedStepIndex = Math.max(0, Math.min(payload.currentStepIndex, Math.max(0, nextTotal - 1)));
|
|
3795
|
+
stepIndexRef.current = clampedStepIndex;
|
|
3796
|
+
setCurrentStepIndex(clampedStepIndex);
|
|
3797
|
+
if (nextTotal > 0) {
|
|
3798
|
+
onStepChangeRef.current?.(clampedStepIndex, nextTotal, updatedTour);
|
|
3804
3799
|
}
|
|
3805
|
-
}
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
}
|
|
3818
|
-
}
|
|
3800
|
+
}
|
|
3801
|
+
});
|
|
3802
|
+
socket.on("tour:end", () => {
|
|
3803
|
+
setServerState((prev) => prev ? { ...prev, isActive: false, phase: "completed" } : prev);
|
|
3804
|
+
handleTourEnd();
|
|
3805
|
+
});
|
|
3806
|
+
socket.on("tour:debug_log", (entry) => {
|
|
3807
|
+
const isDev = devModeRef.current || process.env.NODE_ENV === "development" || typeof window !== "undefined" && window.MODELNEX_DEBUG;
|
|
3808
|
+
if (isDev) {
|
|
3809
|
+
console.log(`%c[ModelNex Tour] ${entry.type}`, "color: #3b82f6; font-weight: bold", entry);
|
|
3810
|
+
if (typeof window !== "undefined") {
|
|
3811
|
+
window.dispatchEvent(new CustomEvent("modelnex-debug", {
|
|
3812
|
+
detail: { msg: `[Tour Timeline] ${entry.type}`, data: entry }
|
|
3813
|
+
}));
|
|
3819
3814
|
}
|
|
3820
|
-
}
|
|
3821
|
-
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3815
|
+
}
|
|
3822
3816
|
});
|
|
3817
|
+
console.log("[ModelNex SDK] Tour playback initialized. Debug logs enabled:", devModeRef.current || process.env.NODE_ENV === "development");
|
|
3823
3818
|
return () => {
|
|
3824
|
-
cancelled = true;
|
|
3825
3819
|
const toClose = createdSocket ?? socketRef.current;
|
|
3826
3820
|
if (toClose) {
|
|
3827
3821
|
toClose.disconnect();
|
|
@@ -3838,7 +3832,12 @@ function useTourPlayback({
|
|
|
3838
3832
|
const s = socketRef.current;
|
|
3839
3833
|
const profile = userProfile;
|
|
3840
3834
|
if (!s?.connected || !websiteId || !profile?.userId) return;
|
|
3841
|
-
|
|
3835
|
+
const timer = setTimeout(() => {
|
|
3836
|
+
if (s.connected) {
|
|
3837
|
+
s.emit("tour:init", { websiteId, userId: profile.userId, userType: profile.type });
|
|
3838
|
+
}
|
|
3839
|
+
}, 150);
|
|
3840
|
+
return () => clearTimeout(timer);
|
|
3842
3841
|
}, [disabled, websiteId, userProfile?.userId, userProfile?.type]);
|
|
3843
3842
|
(0, import_react12.useEffect)(() => {
|
|
3844
3843
|
if (!showCaptions || !isReviewMode) {
|
|
@@ -3863,11 +3862,13 @@ function useTourPlayback({
|
|
|
3863
3862
|
if (!socketRef.current?.connected || !isActiveRef.current) return;
|
|
3864
3863
|
const { generateMinifiedAOM: generateMinifiedAOM2 } = await Promise.resolve().then(() => (init_aom(), aom_exports));
|
|
3865
3864
|
const aom = generateMinifiedAOM2();
|
|
3866
|
-
socketRef.current
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3865
|
+
if (socketRef.current?.connected) {
|
|
3866
|
+
socketRef.current.emit("tour:sync_dom", {
|
|
3867
|
+
url: window.location.pathname + window.location.search + window.location.hash,
|
|
3868
|
+
aom: aom.nodes,
|
|
3869
|
+
domSummary: captureDomSummary()
|
|
3870
|
+
});
|
|
3871
|
+
}
|
|
3871
3872
|
}, []);
|
|
3872
3873
|
const interruptExecution = (0, import_react12.useCallback)((transcript) => {
|
|
3873
3874
|
if (!socketRef.current?.connected || !isActiveRef.current) return false;
|
|
@@ -3878,21 +3879,23 @@ function useTourPlayback({
|
|
|
3878
3879
|
removeHighlight();
|
|
3879
3880
|
removeCaption();
|
|
3880
3881
|
voice.stopSpeaking();
|
|
3881
|
-
socketRef.current
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3882
|
+
if (socketRef.current?.connected) {
|
|
3883
|
+
socketRef.current.emit("tour:action_result", {
|
|
3884
|
+
success: true,
|
|
3885
|
+
interrupted: true,
|
|
3886
|
+
transcript,
|
|
3887
|
+
commandBatchId: activeCommandBatchIdRef.current,
|
|
3888
|
+
runId: runIdRef.current,
|
|
3889
|
+
turnId: turnIdRef.current
|
|
3890
|
+
});
|
|
3891
|
+
activeCommandBatchIdRef.current = null;
|
|
3892
|
+
socketRef.current.emit("tour:user_input", {
|
|
3893
|
+
transcript,
|
|
3894
|
+
interrupted: true,
|
|
3895
|
+
runId: runIdRef.current,
|
|
3896
|
+
turnId: turnIdRef.current
|
|
3897
|
+
});
|
|
3898
|
+
}
|
|
3896
3899
|
setPlaybackState("thinking");
|
|
3897
3900
|
return true;
|
|
3898
3901
|
}, [voice]);
|
|
@@ -4010,11 +4013,13 @@ function useTourPlayback({
|
|
|
4010
4013
|
previewRunIdRef.current = null;
|
|
4011
4014
|
}
|
|
4012
4015
|
tourRef.current = tour;
|
|
4013
|
-
socketRef.current
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4016
|
+
if (socketRef.current?.connected) {
|
|
4017
|
+
socketRef.current.emit("tour:request_start", {
|
|
4018
|
+
tourId: tour.id,
|
|
4019
|
+
previewRunId: previewRunIdRef.current,
|
|
4020
|
+
tourContext: tour
|
|
4021
|
+
});
|
|
4022
|
+
}
|
|
4018
4023
|
}, [serverUrl, websiteId]);
|
|
4019
4024
|
(0, import_react12.useEffect)(() => {
|
|
4020
4025
|
if (!enableAutoDiscovery) return;
|
|
@@ -5462,7 +5467,7 @@ function useVoice(serverUrl) {
|
|
|
5462
5467
|
(async () => {
|
|
5463
5468
|
try {
|
|
5464
5469
|
const ioModule = await import("socket.io-client");
|
|
5465
|
-
const
|
|
5470
|
+
const io3 = ioModule.default || ioModule;
|
|
5466
5471
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
5467
5472
|
audio: {
|
|
5468
5473
|
echoCancellation: true,
|
|
@@ -5484,7 +5489,7 @@ function useVoice(serverUrl) {
|
|
|
5484
5489
|
audioBitsPerSecond: 128e3
|
|
5485
5490
|
});
|
|
5486
5491
|
mediaRecorderRef.current = recorder;
|
|
5487
|
-
const socket =
|
|
5492
|
+
const socket = io3(serverUrl, {
|
|
5488
5493
|
path: "/socket.io",
|
|
5489
5494
|
transports: resolveSocketIoTransports(serverUrl, "websocket-first")
|
|
5490
5495
|
});
|
|
@@ -11005,7 +11010,7 @@ var ModelNexProvider = ({
|
|
|
11005
11010
|
socketId,
|
|
11006
11011
|
devMode
|
|
11007
11012
|
}),
|
|
11008
|
-
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile, toursApiBase, voiceMuted, socketId, devMode]
|
|
11013
|
+
[serverUrl, commandUrl, registerAction, unregisterAction, activeAgentActions, stagingFields, highlightActions, studioMode, recordingMode, extractedElements, tagStore, chatMessages, websiteId, userProfile?.userId, userProfile?.type, userProfile?.isNewUser, toursApiBase, voiceMuted, socketId, devMode]
|
|
11009
11014
|
);
|
|
11010
11015
|
return import_react21.default.createElement(
|
|
11011
11016
|
ModelNexContext.Provider,
|