@midscene/visualizer 0.28.10-beta-20250919084614.0 → 0.28.10-beta-20250922071252.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/component/player/index.mjs +15 -38
- package/dist/es/component/universal-playground/index.css +1 -1
- package/dist/es/component/universal-playground/index.mjs +5 -29
- package/dist/es/component/universal-playground/providers/storage-provider.mjs +3 -42
- package/dist/es/hooks/usePlaygroundState.mjs +3 -28
- package/dist/es/index.mjs +2 -2
- package/dist/es/utils/replay-scripts.mjs +32 -59
- package/dist/lib/component/player/index.js +15 -38
- package/dist/lib/component/universal-playground/index.css +1 -1
- package/dist/lib/component/universal-playground/index.js +3 -27
- package/dist/lib/component/universal-playground/providers/storage-provider.js +3 -60
- package/dist/lib/hooks/usePlaygroundState.js +3 -28
- package/dist/lib/index.js +2 -14
- package/dist/lib/utils/replay-scripts.js +32 -59
- package/dist/types/component/universal-playground/providers/storage-provider.d.ts +0 -16
- package/dist/types/hooks/usePlaygroundExecution.d.ts +1 -1
- package/dist/types/hooks/usePlaygroundState.d.ts +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/utils/replay-scripts.d.ts +0 -2
- package/package.json +5 -5
- package/dist/es/component/universal-playground/providers/indexeddb-storage-provider.mjs +0 -207
- package/dist/lib/component/universal-playground/providers/indexeddb-storage-provider.js +0 -247
- package/dist/types/component/universal-playground/providers/indexeddb-storage-provider.d.ts +0 -71
|
@@ -172,29 +172,9 @@ function Player(props) {
|
|
|
172
172
|
const cameraState = useRef({
|
|
173
173
|
...basicCameraState
|
|
174
174
|
});
|
|
175
|
-
const
|
|
176
|
-
if (app.screen.width !== newWidth || app.screen.height !== newHeight) {
|
|
177
|
-
app.renderer.resize(newWidth, newHeight);
|
|
178
|
-
if (divContainerRef.current) {
|
|
179
|
-
const aspectRatio = newWidth / newHeight;
|
|
180
|
-
divContainerRef.current.style.setProperty('--canvas-aspect-ratio', aspectRatio.toString());
|
|
181
|
-
}
|
|
182
|
-
const newBasicCameraState = {
|
|
183
|
-
left: 0,
|
|
184
|
-
top: 0,
|
|
185
|
-
width: newWidth,
|
|
186
|
-
pointerLeft: Math.round(newWidth / 2),
|
|
187
|
-
pointerTop: Math.round(newHeight / 2)
|
|
188
|
-
};
|
|
189
|
-
cameraState.current = newBasicCameraState;
|
|
190
|
-
}
|
|
191
|
-
};
|
|
192
|
-
const repaintImage = async (scriptWidth, scriptHeight)=>{
|
|
175
|
+
const repaintImage = async ()=>{
|
|
193
176
|
const imgToUpdate = currentImg.current;
|
|
194
177
|
if (!imgToUpdate) return void console.warn('no image to update');
|
|
195
|
-
const targetWidth = scriptWidth || imageWidth;
|
|
196
|
-
const targetHeight = scriptHeight || imageHeight;
|
|
197
|
-
await resizeCanvasIfNeeded(targetWidth, targetHeight);
|
|
198
178
|
if (!getTextureFromCache(imgToUpdate)) {
|
|
199
179
|
console.warn('image not loaded', imgToUpdate);
|
|
200
180
|
await loadTexture(imgToUpdate);
|
|
@@ -208,8 +188,8 @@ function Player(props) {
|
|
|
208
188
|
if (child) windowContentContainer.removeChild(child);
|
|
209
189
|
sprite.label = mainImgLabel;
|
|
210
190
|
sprite.zIndex = LAYER_ORDER_IMG;
|
|
211
|
-
sprite.width =
|
|
212
|
-
sprite.height =
|
|
191
|
+
sprite.width = imageWidth;
|
|
192
|
+
sprite.height = imageHeight;
|
|
213
193
|
windowContentContainer.addChild(sprite);
|
|
214
194
|
};
|
|
215
195
|
const spinningPointer = (frame)=>{
|
|
@@ -267,10 +247,9 @@ function Player(props) {
|
|
|
267
247
|
pointerSprite.current.zIndex = LAYER_ORDER_POINTER;
|
|
268
248
|
windowContentContainer.addChild(pointerSprite.current);
|
|
269
249
|
};
|
|
270
|
-
const updateCamera = (state
|
|
250
|
+
const updateCamera = (state)=>{
|
|
271
251
|
cameraState.current = state;
|
|
272
|
-
const
|
|
273
|
-
const newScale = autoZoom ? Math.max(1, effectiveWidth / state.width) : 1;
|
|
252
|
+
const newScale = autoZoom ? Math.max(1, imageWidth / state.width) : 1;
|
|
274
253
|
windowContentContainer.scale.set(newScale);
|
|
275
254
|
windowContentContainer.x = autoZoom ? Math.round(canvasPaddingLeft - state.left * newScale) : canvasPaddingLeft;
|
|
276
255
|
windowContentContainer.y = autoZoom ? Math.round(canvasPaddingTop - state.top * newScale) : canvasPaddingTop;
|
|
@@ -284,8 +263,6 @@ function Player(props) {
|
|
|
284
263
|
}
|
|
285
264
|
};
|
|
286
265
|
const cameraAnimation = async (targetState, duration, frame)=>{
|
|
287
|
-
const currentCanvasWidth = app.screen.width || imageWidth;
|
|
288
|
-
const currentCanvasHeight = app.screen.height || imageHeight;
|
|
289
266
|
if (!autoZoom) {
|
|
290
267
|
const currentState = {
|
|
291
268
|
...cameraState.current
|
|
@@ -305,7 +282,7 @@ function Player(props) {
|
|
|
305
282
|
pointerLeft: startPointerLeft + (targetState.pointerLeft - startPointerLeft) * progress,
|
|
306
283
|
pointerTop: startPointerTop + (targetState.pointerTop - startPointerTop) * progress
|
|
307
284
|
};
|
|
308
|
-
updateCamera(nextState
|
|
285
|
+
updateCamera(nextState);
|
|
309
286
|
if (elapsedTime < duration) frame(animate);
|
|
310
287
|
else resolve();
|
|
311
288
|
};
|
|
@@ -320,7 +297,7 @@ function Player(props) {
|
|
|
320
297
|
const startTop = currentState.top;
|
|
321
298
|
const startPointerLeft = currentState.pointerLeft;
|
|
322
299
|
const startPointerTop = currentState.pointerTop;
|
|
323
|
-
const startScale = currentState.width /
|
|
300
|
+
const startScale = currentState.width / imageWidth;
|
|
324
301
|
const startTime = performance.now();
|
|
325
302
|
const shouldMovePointer = 'number' == typeof targetState.pointerLeft && 'number' == typeof targetState.pointerTop && (targetState.pointerLeft !== startPointerLeft || targetState.pointerTop !== startPointerTop);
|
|
326
303
|
const pointerMoveDuration = shouldMovePointer ? 0.375 * duration : 0;
|
|
@@ -345,19 +322,19 @@ function Player(props) {
|
|
|
345
322
|
const cameraElapsedTime = elapsedTime - cameraMoveStart;
|
|
346
323
|
const rawCameraProgress = Math.min(cameraElapsedTime / cameraMoveDuration, 1);
|
|
347
324
|
const cameraProgress = cubicImage(rawCameraProgress);
|
|
348
|
-
const targetScale = targetState.width /
|
|
325
|
+
const targetScale = targetState.width / imageWidth;
|
|
349
326
|
const progressScale = startScale + (targetScale - startScale) * cameraProgress;
|
|
350
|
-
const progressWidth =
|
|
351
|
-
const progressHeight =
|
|
327
|
+
const progressWidth = imageWidth * progressScale;
|
|
328
|
+
const progressHeight = imageHeight * progressScale;
|
|
352
329
|
nextState.width = progressWidth;
|
|
353
330
|
const progressLeft = startLeft + (targetState.left - startLeft) * cameraProgress;
|
|
354
331
|
const progressTop = startTop + (targetState.top - startTop) * cameraProgress;
|
|
355
|
-
const horizontalExceed = progressLeft + progressWidth -
|
|
356
|
-
const verticalExceed = progressTop + progressHeight -
|
|
332
|
+
const horizontalExceed = progressLeft + progressWidth - imageWidth;
|
|
333
|
+
const verticalExceed = progressTop + progressHeight - imageHeight;
|
|
357
334
|
nextState.left = horizontalExceed > 0 ? progressLeft + horizontalExceed : progressLeft;
|
|
358
335
|
nextState.top = verticalExceed > 0 ? progressTop + verticalExceed : progressTop;
|
|
359
336
|
}
|
|
360
|
-
updateCamera(nextState
|
|
337
|
+
updateCamera(nextState);
|
|
361
338
|
if (elapsedTime < duration) frame(animate);
|
|
362
339
|
else resolve();
|
|
363
340
|
};
|
|
@@ -494,7 +471,7 @@ function Player(props) {
|
|
|
494
471
|
var _item_context;
|
|
495
472
|
if (!item.img) throw new Error('img is required');
|
|
496
473
|
currentImg.current = item.img;
|
|
497
|
-
await repaintImage(
|
|
474
|
+
await repaintImage();
|
|
498
475
|
const elements = (null == (_item_context = item.context) ? void 0 : _item_context.tree) ? treeToList(item.context.tree) : [];
|
|
499
476
|
const highlightElements = item.highlightElement ? [
|
|
500
477
|
item.highlightElement
|
|
@@ -511,7 +488,7 @@ function Player(props) {
|
|
|
511
488
|
} else if ('img' === item.type) {
|
|
512
489
|
if (item.img && item.img !== currentImg.current) {
|
|
513
490
|
currentImg.current = item.img;
|
|
514
|
-
await repaintImage(
|
|
491
|
+
await repaintImage();
|
|
515
492
|
}
|
|
516
493
|
if (item.camera) await cameraAnimation(item.camera, item.duration, frame);
|
|
517
494
|
else await sleep(item.duration);
|
|
@@ -91,10 +91,10 @@
|
|
|
91
91
|
.playground-container .middle-dialog-area .scroll-to-bottom-button {
|
|
92
92
|
z-index: 10;
|
|
93
93
|
background: #fff;
|
|
94
|
-
border: 1px solid rgba(0, 0, 0, .08);
|
|
95
94
|
position: absolute;
|
|
96
95
|
bottom: 10px;
|
|
97
96
|
right: 0;
|
|
97
|
+
box-shadow: 0 4px 8px rgba(0, 0, 0, .04);
|
|
98
98
|
}
|
|
99
99
|
|
|
100
100
|
.playground-container .middle-dialog-area .scroll-to-bottom-button:hover {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Fragment, jsx, jsxs } from "react/jsx-runtime";
|
|
2
2
|
import icons, { ArrowDownOutlined, ClearOutlined, LoadingOutlined } from "@ant-design/icons";
|
|
3
3
|
import { Button, Form, List, Tooltip, Typography, message } from "antd";
|
|
4
|
-
import { useCallback, useEffect
|
|
4
|
+
import { useCallback, useEffect } from "react";
|
|
5
5
|
import { usePlaygroundExecution } from "../../hooks/usePlaygroundExecution.mjs";
|
|
6
6
|
import { usePlaygroundState } from "../../hooks/usePlaygroundState.mjs";
|
|
7
7
|
import { useEnvConfig } from "../../store/store.mjs";
|
|
@@ -11,7 +11,7 @@ import { PlaygroundResultView } from "../playground-result/index.mjs";
|
|
|
11
11
|
import "./index.css";
|
|
12
12
|
import avatar from "../../icons/avatar.mjs";
|
|
13
13
|
import { PromptInput } from "../prompt-input/index.mjs";
|
|
14
|
-
import {
|
|
14
|
+
import { LocalStorageProvider } from "./providers/storage-provider.mjs";
|
|
15
15
|
const { Text } = Typography;
|
|
16
16
|
function getSDKId(sdk) {
|
|
17
17
|
if (sdk.id && 'string' == typeof sdk.id) return `agent-${sdk.id}`;
|
|
@@ -41,35 +41,11 @@ function UniversalPlayground(param) {
|
|
|
41
41
|
let { playgroundSDK, storage, contextProvider, config: componentConfig = {}, branding = {}, className = '', dryMode = false, showContextPreview = true } = param;
|
|
42
42
|
const [form] = Form.useForm();
|
|
43
43
|
const { config } = useEnvConfig();
|
|
44
|
-
const
|
|
45
|
-
useEffect(()=>{
|
|
46
|
-
const initializeSDK = async ()=>{
|
|
47
|
-
if (playgroundSDK && 'function' == typeof playgroundSDK.checkStatus) try {
|
|
48
|
-
await playgroundSDK.checkStatus();
|
|
49
|
-
setSdkReady(true);
|
|
50
|
-
} catch (error) {
|
|
51
|
-
console.warn('Failed to initialize SDK, using default namespace:', error);
|
|
52
|
-
setSdkReady(true);
|
|
53
|
-
}
|
|
54
|
-
else setSdkReady(true);
|
|
55
|
-
};
|
|
56
|
-
initializeSDK();
|
|
57
|
-
}, [
|
|
58
|
-
playgroundSDK
|
|
59
|
-
]);
|
|
60
|
-
const effectiveStorage = useMemo(()=>{
|
|
44
|
+
const effectiveStorage = (()=>{
|
|
61
45
|
if (storage) return storage;
|
|
62
|
-
if (!sdkReady) return null;
|
|
63
46
|
const namespace = componentConfig.storageNamespace || getSDKId(playgroundSDK);
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return createStorageProvider(bestStorageType, namespace);
|
|
67
|
-
}, [
|
|
68
|
-
storage,
|
|
69
|
-
sdkReady,
|
|
70
|
-
componentConfig.storageNamespace,
|
|
71
|
-
playgroundSDK
|
|
72
|
-
]);
|
|
47
|
+
return new LocalStorageProvider(namespace);
|
|
48
|
+
})();
|
|
73
49
|
const { loading, setLoading, infoList, setInfoList, actionSpace, actionSpaceLoading, uiContextPreview, setUiContextPreview, showScrollToBottomButton, verticalMode, replayCounter, setReplayCounter, infoListRef, currentRunningIdRef, interruptedFlagRef, clearInfoList, handleScrollToBottom } = usePlaygroundState(playgroundSDK, effectiveStorage, contextProvider);
|
|
74
50
|
const { handleRun: executeAction, handleStop, canStop } = usePlaygroundExecution(playgroundSDK, effectiveStorage, actionSpace, loading, setLoading, infoList, setInfoList, replayCounter, setReplayCounter, verticalMode, currentRunningIdRef, interruptedFlagRef);
|
|
75
51
|
useEffect(()=>{
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { IndexedDBStorageProvider, MemoryStorageProvider, NoOpStorageProvider } from "./indexeddb-storage-provider.mjs";
|
|
2
1
|
function _define_property(obj, key, value) {
|
|
3
2
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
4
3
|
value: value,
|
|
@@ -138,7 +137,7 @@ class LocalStorageProvider {
|
|
|
138
137
|
this.resultsKey = `${namespace}-results`;
|
|
139
138
|
}
|
|
140
139
|
}
|
|
141
|
-
class
|
|
140
|
+
class MemoryStorageProvider {
|
|
142
141
|
async saveMessages(messages) {
|
|
143
142
|
this.messages = [
|
|
144
143
|
...messages
|
|
@@ -161,7 +160,7 @@ class storage_provider_MemoryStorageProvider {
|
|
|
161
160
|
_define_property(this, "results", new Map());
|
|
162
161
|
}
|
|
163
162
|
}
|
|
164
|
-
class
|
|
163
|
+
class NoOpStorageProvider {
|
|
165
164
|
async saveMessages(_messages) {}
|
|
166
165
|
async loadMessages() {
|
|
167
166
|
return [];
|
|
@@ -169,42 +168,4 @@ class storage_provider_NoOpStorageProvider {
|
|
|
169
168
|
async clearMessages() {}
|
|
170
169
|
async saveResult(_id, _result) {}
|
|
171
170
|
}
|
|
172
|
-
|
|
173
|
-
StorageType["INDEXEDDB"] = "indexeddb";
|
|
174
|
-
StorageType["LOCALSTORAGE"] = "localStorage";
|
|
175
|
-
StorageType["MEMORY"] = "memory";
|
|
176
|
-
StorageType["NONE"] = "none";
|
|
177
|
-
return StorageType;
|
|
178
|
-
}({});
|
|
179
|
-
function createStorageProvider() {
|
|
180
|
-
let type = arguments.length > 0 && void 0 !== arguments[0] ? arguments[0] : "indexeddb", namespace = arguments.length > 1 && void 0 !== arguments[1] ? arguments[1] : 'playground';
|
|
181
|
-
switch(type){
|
|
182
|
-
case "indexeddb":
|
|
183
|
-
if ('undefined' != typeof indexedDB) return new IndexedDBStorageProvider(namespace);
|
|
184
|
-
console.warn('IndexedDB not available, falling back to localStorage');
|
|
185
|
-
return createStorageProvider("localStorage", namespace);
|
|
186
|
-
case "localStorage":
|
|
187
|
-
if ('undefined' != typeof localStorage) return new LocalStorageProvider(namespace);
|
|
188
|
-
console.warn('localStorage not available, falling back to memory storage');
|
|
189
|
-
return createStorageProvider("memory", namespace);
|
|
190
|
-
case "memory":
|
|
191
|
-
return new storage_provider_MemoryStorageProvider();
|
|
192
|
-
case "none":
|
|
193
|
-
return new storage_provider_NoOpStorageProvider();
|
|
194
|
-
default:
|
|
195
|
-
throw new Error(`Unknown storage type: ${type}`);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
function detectBestStorageType() {
|
|
199
|
-
if ('undefined' != typeof indexedDB) try {
|
|
200
|
-
indexedDB.open('test', 1).onerror = ()=>{};
|
|
201
|
-
return "indexeddb";
|
|
202
|
-
} catch (e) {}
|
|
203
|
-
if ('undefined' != typeof localStorage) try {
|
|
204
|
-
localStorage.setItem('test', 'test');
|
|
205
|
-
localStorage.removeItem('test');
|
|
206
|
-
return "localStorage";
|
|
207
|
-
} catch (e) {}
|
|
208
|
-
return "memory";
|
|
209
|
-
}
|
|
210
|
-
export { MemoryStorageProvider as IndexedDBMemoryStorageProvider, NoOpStorageProvider as IndexedDBNoOpStorageProvider, IndexedDBStorageProvider, LocalStorageProvider, storage_provider_MemoryStorageProvider as MemoryStorageProvider, storage_provider_NoOpStorageProvider as NoOpStorageProvider, storage_provider_StorageType as StorageType, createStorageProvider, detectBestStorageType };
|
|
171
|
+
export { LocalStorageProvider, MemoryStorageProvider, NoOpStorageProvider };
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { useCallback, useEffect, useRef, useState } from "react";
|
|
2
|
-
import { createStorageProvider, detectBestStorageType } from "../component/universal-playground/providers/storage-provider.mjs";
|
|
3
2
|
import { WELCOME_MESSAGE_TEMPLATE } from "../utils/constants.mjs";
|
|
4
3
|
function usePlaygroundState(playgroundSDK, storage, contextProvider) {
|
|
5
4
|
const [loading, setLoading] = useState(false);
|
|
@@ -13,25 +12,7 @@ function usePlaygroundState(playgroundSDK, storage, contextProvider) {
|
|
|
13
12
|
const infoListRef = useRef(null);
|
|
14
13
|
const currentRunningIdRef = useRef(null);
|
|
15
14
|
const interruptedFlagRef = useRef({});
|
|
16
|
-
const initializedRef = useRef(false);
|
|
17
15
|
useEffect(()=>{
|
|
18
|
-
const migrateFromOldNamespace = async ()=>{
|
|
19
|
-
const oldStorage = createStorageProvider(detectBestStorageType(), 'playground-default');
|
|
20
|
-
try {
|
|
21
|
-
if (null == oldStorage ? void 0 : oldStorage.loadMessages) {
|
|
22
|
-
const oldMessages = await oldStorage.loadMessages();
|
|
23
|
-
if (oldMessages.length > 1) {
|
|
24
|
-
console.log('Found data in old namespace, migrating...');
|
|
25
|
-
if (null == storage ? void 0 : storage.saveMessages) await storage.saveMessages(oldMessages);
|
|
26
|
-
if (oldStorage.clearMessages) await oldStorage.clearMessages();
|
|
27
|
-
return oldMessages;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
} catch (error) {
|
|
31
|
-
console.debug('No data found in old namespace:', error);
|
|
32
|
-
}
|
|
33
|
-
return [];
|
|
34
|
-
};
|
|
35
16
|
const initializeMessages = async ()=>{
|
|
36
17
|
const welcomeMessage = {
|
|
37
18
|
...WELCOME_MESSAGE_TEMPLATE,
|
|
@@ -39,8 +20,7 @@ function usePlaygroundState(playgroundSDK, storage, contextProvider) {
|
|
|
39
20
|
timestamp: new Date()
|
|
40
21
|
};
|
|
41
22
|
if (null == storage ? void 0 : storage.loadMessages) try {
|
|
42
|
-
|
|
43
|
-
if (0 === storedMessages.length) storedMessages = await migrateFromOldNamespace();
|
|
23
|
+
const storedMessages = await storage.loadMessages();
|
|
44
24
|
const hasWelcomeMessage = storedMessages.some((msg)=>'welcome' === msg.id);
|
|
45
25
|
hasWelcomeMessage ? setInfoList(storedMessages) : setInfoList([
|
|
46
26
|
welcomeMessage,
|
|
@@ -56,13 +36,8 @@ function usePlaygroundState(playgroundSDK, storage, contextProvider) {
|
|
|
56
36
|
welcomeMessage
|
|
57
37
|
]);
|
|
58
38
|
};
|
|
59
|
-
if (
|
|
60
|
-
|
|
61
|
-
initializeMessages();
|
|
62
|
-
} else if (!storage && 0 === infoList.length) initializeMessages();
|
|
63
|
-
}, [
|
|
64
|
-
storage
|
|
65
|
-
]);
|
|
39
|
+
if (0 === infoList.length) initializeMessages();
|
|
40
|
+
}, []);
|
|
66
41
|
useEffect(()=>{
|
|
67
42
|
if ((null == storage ? void 0 : storage.saveMessages) && infoList.length > 1) storage.saveMessages(infoList).catch((error)=>{
|
|
68
43
|
if (error instanceof DOMException && 'QuotaExceededError' === error.name) console.warn('Storage quota exceeded - some messages may not be saved persistently');
|
package/dist/es/index.mjs
CHANGED
|
@@ -20,6 +20,6 @@ import { actionNameForType, getPlaceholderForType, staticAgentFromContext } from
|
|
|
20
20
|
import { filterBase64Value, timeStr } from "./utils/index.mjs";
|
|
21
21
|
import shiny_text from "./component/shiny-text/index.mjs";
|
|
22
22
|
import universal_playground, { UniversalPlayground } from "./component/universal-playground/index.mjs";
|
|
23
|
-
import {
|
|
23
|
+
import { LocalStorageProvider, MemoryStorageProvider, NoOpStorageProvider } from "./component/universal-playground/providers/storage-provider.mjs";
|
|
24
24
|
import { AgentContextProvider, BaseContextProvider, NoOpContextProvider, StaticContextProvider } from "./component/universal-playground/providers/context-provider.mjs";
|
|
25
|
-
export { AgentContextProvider, BaseContextProvider, Blackboard, ContextPreview, EnvConfig, EnvConfigReminder,
|
|
25
|
+
export { AgentContextProvider, BaseContextProvider, Blackboard, ContextPreview, EnvConfig, EnvConfigReminder, LocalStorageProvider, Logo, MemoryStorageProvider, NavActions, NoOpContextProvider, NoOpStorageProvider, Player, PlaygroundResultView, PromptInput, ServiceModeControl, shiny_text as ShinyText, StaticContextProvider, UniversalPlayground, universal_playground as UniversalPlaygroundDefault, actionNameForType, allScriptsFromDump, colorForName, filterBase64Value, generateAnimationScripts, getPlaceholderForType, globalThemeConfig, highlightColorForType, iconForStatus, safeOverrideAIConfig, staticAgentFromContext, timeCostStrElement, timeStr, useEnvConfig, useSafeOverrideAIConfig, useServerValid };
|
|
@@ -42,9 +42,8 @@ const capitalizeFirstLetter = (str)=>{
|
|
|
42
42
|
return str.charAt(0).toUpperCase() + str.slice(1);
|
|
43
43
|
};
|
|
44
44
|
const allScriptsFromDump = (dump)=>{
|
|
45
|
-
|
|
46
|
-
let
|
|
47
|
-
let firstHeight;
|
|
45
|
+
let width;
|
|
46
|
+
let height;
|
|
48
47
|
let sdkVersion;
|
|
49
48
|
const modelBriefsSet = new Set();
|
|
50
49
|
dump.executions.forEach((execution)=>{
|
|
@@ -52,17 +51,12 @@ const allScriptsFromDump = (dump)=>{
|
|
|
52
51
|
execution.tasks.forEach((task)=>{
|
|
53
52
|
var _task_uiContext_size, _task_uiContext;
|
|
54
53
|
if (null == (_task_uiContext = task.uiContext) ? void 0 : null == (_task_uiContext_size = _task_uiContext.size) ? void 0 : _task_uiContext_size.width) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
if (!firstWidth) {
|
|
58
|
-
firstWidth = w;
|
|
59
|
-
firstHeight = h;
|
|
60
|
-
}
|
|
61
|
-
dimensionsSet.add(`${w}x${h}`);
|
|
54
|
+
width = task.uiContext.size.width;
|
|
55
|
+
height = task.uiContext.size.height;
|
|
62
56
|
}
|
|
63
57
|
});
|
|
64
58
|
});
|
|
65
|
-
if (!
|
|
59
|
+
if (!width || !height) {
|
|
66
60
|
console.warn('width or height is missing in dump file');
|
|
67
61
|
return {
|
|
68
62
|
scripts: [],
|
|
@@ -72,7 +66,7 @@ const allScriptsFromDump = (dump)=>{
|
|
|
72
66
|
}
|
|
73
67
|
const allScripts = [];
|
|
74
68
|
dump.executions.forEach((execution)=>{
|
|
75
|
-
const scripts = generateAnimationScripts(execution, -1,
|
|
69
|
+
const scripts = generateAnimationScripts(execution, -1, width, height);
|
|
76
70
|
if (scripts) allScripts.push(...scripts);
|
|
77
71
|
execution.tasks.forEach((task)=>{
|
|
78
72
|
if (task.usage) {
|
|
@@ -99,8 +93,8 @@ const allScriptsFromDump = (dump)=>{
|
|
|
99
93
|
})();
|
|
100
94
|
return {
|
|
101
95
|
scripts: allScriptsWithoutIntermediateDoneFrame,
|
|
102
|
-
width
|
|
103
|
-
height
|
|
96
|
+
width,
|
|
97
|
+
height,
|
|
104
98
|
sdkVersion,
|
|
105
99
|
modelBriefs
|
|
106
100
|
};
|
|
@@ -149,16 +143,14 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
149
143
|
if ('Planning' === task.type) {
|
|
150
144
|
const planningTask = task;
|
|
151
145
|
if (planningTask.recorder && planningTask.recorder.length > 0) {
|
|
152
|
-
var _planningTask_recorder_, _planningTask_recorder
|
|
146
|
+
var _planningTask_recorder_, _planningTask_recorder;
|
|
153
147
|
scripts.push({
|
|
154
148
|
type: 'img',
|
|
155
149
|
img: null == (_planningTask_recorder = planningTask.recorder) ? void 0 : null == (_planningTask_recorder_ = _planningTask_recorder[0]) ? void 0 : _planningTask_recorder_.screenshot,
|
|
156
150
|
camera: 0 === index ? fullPageCameraState : void 0,
|
|
157
151
|
duration: stillDuration,
|
|
158
152
|
title: typeStr(task),
|
|
159
|
-
subTitle: paramStr(task)
|
|
160
|
-
imageWidth: (null == (_task_uiContext = task.uiContext) ? void 0 : null == (_task_uiContext_size = _task_uiContext.size) ? void 0 : _task_uiContext_size.width) || imageWidth,
|
|
161
|
-
imageHeight: (null == (_task_uiContext1 = task.uiContext) ? void 0 : null == (_task_uiContext_size1 = _task_uiContext1.size) ? void 0 : _task_uiContext_size1.height) || imageHeight
|
|
153
|
+
subTitle: paramStr(task)
|
|
162
154
|
});
|
|
163
155
|
}
|
|
164
156
|
} else if ('Insight' === task.type && 'Locate' === task.subType) {
|
|
@@ -174,21 +166,16 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
174
166
|
};
|
|
175
167
|
const context = insightTask.uiContext;
|
|
176
168
|
if (null == context ? void 0 : context.screenshotBase64) {
|
|
177
|
-
var _insightTask_log, _insightTask_output1, _insightDump_taskInfo
|
|
169
|
+
var _insightTask_log, _insightTask_output1, _insightDump_taskInfo;
|
|
178
170
|
const insightDump = null == (_insightTask_log = insightTask.log) ? void 0 : _insightTask_log.dump;
|
|
179
171
|
const insightContentLength = context.tree ? treeToList(context.tree).length : 0;
|
|
180
|
-
if (context.screenshotBase64) {
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
subTitle,
|
|
188
|
-
imageWidth: (null == (_context_size2 = context.size) ? void 0 : _context_size2.width) || imageWidth,
|
|
189
|
-
imageHeight: (null == (_context_size3 = context.size) ? void 0 : _context_size3.height) || imageHeight
|
|
190
|
-
});
|
|
191
|
-
}
|
|
172
|
+
if (context.screenshotBase64) scripts.push({
|
|
173
|
+
type: 'img',
|
|
174
|
+
img: context.screenshotBase64,
|
|
175
|
+
duration: stillAfterInsightDuration,
|
|
176
|
+
title,
|
|
177
|
+
subTitle
|
|
178
|
+
});
|
|
192
179
|
let cameraState;
|
|
193
180
|
cameraState = currentCameraState === fullPageCameraState ? void 0 : insightCameraState ? mergeTwoCameraState(currentCameraState, insightCameraState) : void 0;
|
|
194
181
|
scripts.push({
|
|
@@ -201,9 +188,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
201
188
|
duration: insightContentLength > 20 ? locateDuration : 0.5 * locateDuration,
|
|
202
189
|
insightCameraDuration: locateDuration,
|
|
203
190
|
title,
|
|
204
|
-
subTitle
|
|
205
|
-
imageWidth: (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth,
|
|
206
|
-
imageHeight: (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight
|
|
191
|
+
subTitle
|
|
207
192
|
});
|
|
208
193
|
scripts.push({
|
|
209
194
|
type: 'sleep',
|
|
@@ -214,7 +199,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
214
199
|
insightOnTop = true;
|
|
215
200
|
}
|
|
216
201
|
} else if ('Action' === task.type && 'FalsyConditionStatement' !== task.subType) {
|
|
217
|
-
var _task_recorder_, _task_recorder,
|
|
202
|
+
var _task_recorder_, _task_recorder, _task_recorder_1, _task_recorder1;
|
|
218
203
|
const title = typeStr(task);
|
|
219
204
|
const subTitle = paramStr(task);
|
|
220
205
|
scripts.push(pointerScript(mousePointer, title, subTitle));
|
|
@@ -225,9 +210,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
225
210
|
duration: actionDuration,
|
|
226
211
|
camera: 'Sleep' === task.subType ? fullPageCameraState : insightCameraState,
|
|
227
212
|
title,
|
|
228
|
-
subTitle
|
|
229
|
-
imageWidth: (null == (_task_uiContext2 = task.uiContext) ? void 0 : null == (_task_uiContext_size2 = _task_uiContext2.size) ? void 0 : _task_uiContext_size2.width) || imageWidth,
|
|
230
|
-
imageHeight: (null == (_task_uiContext3 = task.uiContext) ? void 0 : null == (_task_uiContext_size3 = _task_uiContext3.size) ? void 0 : _task_uiContext_size3.height) || imageHeight
|
|
213
|
+
subTitle
|
|
231
214
|
});
|
|
232
215
|
if (insightOnTop) {
|
|
233
216
|
scripts.push({
|
|
@@ -240,7 +223,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
240
223
|
}
|
|
241
224
|
const imgStillDuration = index < taskCount - 1 ? stillDuration : 0;
|
|
242
225
|
if (null == (_task_recorder1 = task.recorder) ? void 0 : null == (_task_recorder_1 = _task_recorder1[1]) ? void 0 : _task_recorder_1.screenshot) {
|
|
243
|
-
var _task_recorder_2, _task_recorder2
|
|
226
|
+
var _task_recorder_2, _task_recorder2;
|
|
244
227
|
scripts.push({
|
|
245
228
|
type: 'spinning-pointer',
|
|
246
229
|
duration: actionSpinningPointerDuration,
|
|
@@ -253,9 +236,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
253
236
|
img: null == (_task_recorder2 = task.recorder) ? void 0 : null == (_task_recorder_2 = _task_recorder2[1]) ? void 0 : _task_recorder_2.screenshot,
|
|
254
237
|
duration: imgStillDuration,
|
|
255
238
|
title,
|
|
256
|
-
subTitle
|
|
257
|
-
imageWidth: (null == (_task_uiContext4 = task.uiContext) ? void 0 : null == (_task_uiContext_size4 = _task_uiContext4.size) ? void 0 : _task_uiContext_size4.width) || imageWidth,
|
|
258
|
-
imageHeight: (null == (_task_uiContext5 = task.uiContext) ? void 0 : null == (_task_uiContext_size5 = _task_uiContext5.size) ? void 0 : _task_uiContext_size5.height) || imageHeight
|
|
239
|
+
subTitle
|
|
259
240
|
});
|
|
260
241
|
} else scripts.push({
|
|
261
242
|
type: 'sleep',
|
|
@@ -268,22 +249,16 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
268
249
|
const title = typeStr(task);
|
|
269
250
|
const subTitle = paramStr(task);
|
|
270
251
|
const screenshot = null == (_task_recorder3 = task.recorder) ? void 0 : null == (_task_recorder_3 = _task_recorder3[task.recorder.length - 1]) ? void 0 : _task_recorder_3.screenshot;
|
|
271
|
-
if (screenshot) {
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
subTitle,
|
|
280
|
-
imageWidth: (null == (_task_uiContext6 = task.uiContext) ? void 0 : null == (_task_uiContext_size6 = _task_uiContext6.size) ? void 0 : _task_uiContext_size6.width) || imageWidth,
|
|
281
|
-
imageHeight: (null == (_task_uiContext7 = task.uiContext) ? void 0 : null == (_task_uiContext_size7 = _task_uiContext7.size) ? void 0 : _task_uiContext_size7.height) || imageHeight
|
|
282
|
-
});
|
|
283
|
-
}
|
|
252
|
+
if (screenshot) scripts.push({
|
|
253
|
+
type: 'img',
|
|
254
|
+
img: screenshot,
|
|
255
|
+
duration: stillDuration,
|
|
256
|
+
camera: fullPageCameraState,
|
|
257
|
+
title,
|
|
258
|
+
subTitle
|
|
259
|
+
});
|
|
284
260
|
}
|
|
285
261
|
if ('finished' !== task.status) {
|
|
286
|
-
var _task_uiContext_size8, _task_uiContext8, _task_uiContext_size9, _task_uiContext9;
|
|
287
262
|
errorStateFlag = true;
|
|
288
263
|
const errorTitle = typeStr(task);
|
|
289
264
|
const errorMsg = task.errorMessage || 'unknown error';
|
|
@@ -294,9 +269,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
|
|
|
294
269
|
camera: fullPageCameraState,
|
|
295
270
|
duration: stillDuration,
|
|
296
271
|
title: errorTitle,
|
|
297
|
-
subTitle: errorSubTitle
|
|
298
|
-
imageWidth: (null == (_task_uiContext8 = task.uiContext) ? void 0 : null == (_task_uiContext_size8 = _task_uiContext8.size) ? void 0 : _task_uiContext_size8.width) || imageWidth,
|
|
299
|
-
imageHeight: (null == (_task_uiContext9 = task.uiContext) ? void 0 : null == (_task_uiContext_size9 = _task_uiContext9.size) ? void 0 : _task_uiContext_size9.height) || imageHeight
|
|
272
|
+
subTitle: errorSubTitle
|
|
300
273
|
});
|
|
301
274
|
return;
|
|
302
275
|
}
|
|
@@ -212,29 +212,9 @@ function Player(props) {
|
|
|
212
212
|
const cameraState = (0, external_react_namespaceObject.useRef)({
|
|
213
213
|
...basicCameraState
|
|
214
214
|
});
|
|
215
|
-
const
|
|
216
|
-
if (app.screen.width !== newWidth || app.screen.height !== newHeight) {
|
|
217
|
-
app.renderer.resize(newWidth, newHeight);
|
|
218
|
-
if (divContainerRef.current) {
|
|
219
|
-
const aspectRatio = newWidth / newHeight;
|
|
220
|
-
divContainerRef.current.style.setProperty('--canvas-aspect-ratio', aspectRatio.toString());
|
|
221
|
-
}
|
|
222
|
-
const newBasicCameraState = {
|
|
223
|
-
left: 0,
|
|
224
|
-
top: 0,
|
|
225
|
-
width: newWidth,
|
|
226
|
-
pointerLeft: Math.round(newWidth / 2),
|
|
227
|
-
pointerTop: Math.round(newHeight / 2)
|
|
228
|
-
};
|
|
229
|
-
cameraState.current = newBasicCameraState;
|
|
230
|
-
}
|
|
231
|
-
};
|
|
232
|
-
const repaintImage = async (scriptWidth, scriptHeight)=>{
|
|
215
|
+
const repaintImage = async ()=>{
|
|
233
216
|
const imgToUpdate = currentImg.current;
|
|
234
217
|
if (!imgToUpdate) return void console.warn('no image to update');
|
|
235
|
-
const targetWidth = scriptWidth || imageWidth;
|
|
236
|
-
const targetHeight = scriptHeight || imageHeight;
|
|
237
|
-
await resizeCanvasIfNeeded(targetWidth, targetHeight);
|
|
238
218
|
if (!(0, pixi_loader_js_namespaceObject.getTextureFromCache)(imgToUpdate)) {
|
|
239
219
|
console.warn('image not loaded', imgToUpdate);
|
|
240
220
|
await (0, pixi_loader_js_namespaceObject.loadTexture)(imgToUpdate);
|
|
@@ -248,8 +228,8 @@ function Player(props) {
|
|
|
248
228
|
if (child) windowContentContainer.removeChild(child);
|
|
249
229
|
sprite.label = mainImgLabel;
|
|
250
230
|
sprite.zIndex = LAYER_ORDER_IMG;
|
|
251
|
-
sprite.width =
|
|
252
|
-
sprite.height =
|
|
231
|
+
sprite.width = imageWidth;
|
|
232
|
+
sprite.height = imageHeight;
|
|
253
233
|
windowContentContainer.addChild(sprite);
|
|
254
234
|
};
|
|
255
235
|
const spinningPointer = (frame)=>{
|
|
@@ -307,10 +287,9 @@ function Player(props) {
|
|
|
307
287
|
pointerSprite.current.zIndex = LAYER_ORDER_POINTER;
|
|
308
288
|
windowContentContainer.addChild(pointerSprite.current);
|
|
309
289
|
};
|
|
310
|
-
const updateCamera = (state
|
|
290
|
+
const updateCamera = (state)=>{
|
|
311
291
|
cameraState.current = state;
|
|
312
|
-
const
|
|
313
|
-
const newScale = autoZoom ? Math.max(1, effectiveWidth / state.width) : 1;
|
|
292
|
+
const newScale = autoZoom ? Math.max(1, imageWidth / state.width) : 1;
|
|
314
293
|
windowContentContainer.scale.set(newScale);
|
|
315
294
|
windowContentContainer.x = autoZoom ? Math.round(canvasPaddingLeft - state.left * newScale) : canvasPaddingLeft;
|
|
316
295
|
windowContentContainer.y = autoZoom ? Math.round(canvasPaddingTop - state.top * newScale) : canvasPaddingTop;
|
|
@@ -324,8 +303,6 @@ function Player(props) {
|
|
|
324
303
|
}
|
|
325
304
|
};
|
|
326
305
|
const cameraAnimation = async (targetState, duration, frame)=>{
|
|
327
|
-
const currentCanvasWidth = app.screen.width || imageWidth;
|
|
328
|
-
const currentCanvasHeight = app.screen.height || imageHeight;
|
|
329
306
|
if (!autoZoom) {
|
|
330
307
|
const currentState = {
|
|
331
308
|
...cameraState.current
|
|
@@ -345,7 +322,7 @@ function Player(props) {
|
|
|
345
322
|
pointerLeft: startPointerLeft + (targetState.pointerLeft - startPointerLeft) * progress,
|
|
346
323
|
pointerTop: startPointerTop + (targetState.pointerTop - startPointerTop) * progress
|
|
347
324
|
};
|
|
348
|
-
updateCamera(nextState
|
|
325
|
+
updateCamera(nextState);
|
|
349
326
|
if (elapsedTime < duration) frame(animate);
|
|
350
327
|
else resolve();
|
|
351
328
|
};
|
|
@@ -360,7 +337,7 @@ function Player(props) {
|
|
|
360
337
|
const startTop = currentState.top;
|
|
361
338
|
const startPointerLeft = currentState.pointerLeft;
|
|
362
339
|
const startPointerTop = currentState.pointerTop;
|
|
363
|
-
const startScale = currentState.width /
|
|
340
|
+
const startScale = currentState.width / imageWidth;
|
|
364
341
|
const startTime = performance.now();
|
|
365
342
|
const shouldMovePointer = 'number' == typeof targetState.pointerLeft && 'number' == typeof targetState.pointerTop && (targetState.pointerLeft !== startPointerLeft || targetState.pointerTop !== startPointerTop);
|
|
366
343
|
const pointerMoveDuration = shouldMovePointer ? 0.375 * duration : 0;
|
|
@@ -385,19 +362,19 @@ function Player(props) {
|
|
|
385
362
|
const cameraElapsedTime = elapsedTime - cameraMoveStart;
|
|
386
363
|
const rawCameraProgress = Math.min(cameraElapsedTime / cameraMoveDuration, 1);
|
|
387
364
|
const cameraProgress = cubicImage(rawCameraProgress);
|
|
388
|
-
const targetScale = targetState.width /
|
|
365
|
+
const targetScale = targetState.width / imageWidth;
|
|
389
366
|
const progressScale = startScale + (targetScale - startScale) * cameraProgress;
|
|
390
|
-
const progressWidth =
|
|
391
|
-
const progressHeight =
|
|
367
|
+
const progressWidth = imageWidth * progressScale;
|
|
368
|
+
const progressHeight = imageHeight * progressScale;
|
|
392
369
|
nextState.width = progressWidth;
|
|
393
370
|
const progressLeft = startLeft + (targetState.left - startLeft) * cameraProgress;
|
|
394
371
|
const progressTop = startTop + (targetState.top - startTop) * cameraProgress;
|
|
395
|
-
const horizontalExceed = progressLeft + progressWidth -
|
|
396
|
-
const verticalExceed = progressTop + progressHeight -
|
|
372
|
+
const horizontalExceed = progressLeft + progressWidth - imageWidth;
|
|
373
|
+
const verticalExceed = progressTop + progressHeight - imageHeight;
|
|
397
374
|
nextState.left = horizontalExceed > 0 ? progressLeft + horizontalExceed : progressLeft;
|
|
398
375
|
nextState.top = verticalExceed > 0 ? progressTop + verticalExceed : progressTop;
|
|
399
376
|
}
|
|
400
|
-
updateCamera(nextState
|
|
377
|
+
updateCamera(nextState);
|
|
401
378
|
if (elapsedTime < duration) frame(animate);
|
|
402
379
|
else resolve();
|
|
403
380
|
};
|
|
@@ -534,7 +511,7 @@ function Player(props) {
|
|
|
534
511
|
var _item_context;
|
|
535
512
|
if (!item.img) throw new Error('img is required');
|
|
536
513
|
currentImg.current = item.img;
|
|
537
|
-
await repaintImage(
|
|
514
|
+
await repaintImage();
|
|
538
515
|
const elements = (null == (_item_context = item.context) ? void 0 : _item_context.tree) ? (0, extractor_namespaceObject.treeToList)(item.context.tree) : [];
|
|
539
516
|
const highlightElements = item.highlightElement ? [
|
|
540
517
|
item.highlightElement
|
|
@@ -551,7 +528,7 @@ function Player(props) {
|
|
|
551
528
|
} else if ('img' === item.type) {
|
|
552
529
|
if (item.img && item.img !== currentImg.current) {
|
|
553
530
|
currentImg.current = item.img;
|
|
554
|
-
await repaintImage(
|
|
531
|
+
await repaintImage();
|
|
555
532
|
}
|
|
556
533
|
if (item.camera) await cameraAnimation(item.camera, item.duration, frame);
|
|
557
534
|
else await sleep(item.duration);
|