@xopcai/xopc 0.0.368 → 0.0.369
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/_locales/en/messages.json +30 -1
- package/dist/browser-ext/_locales/zh_CN/messages.json +30 -1
- package/dist/browser-ext/dist/assets/{auth-beGtbLPS.js → page-context-BeworyNB.js} +1331 -1355
- package/dist/browser-ext/dist/assets/{sidepanel-BT0ojEzG.css → sidepanel-Ccl0eSa1.css} +43 -0
- package/dist/browser-ext/dist/background.js +680 -182
- package/dist/browser-ext/dist/sidepanel.html +2 -2
- package/dist/browser-ext/dist/sidepanel.js +372 -193
- package/dist/browser-ext/manifest.json +2 -8
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/{activity-store-D9ZMh5ZR.js → activity-store-BQ3PMp2q.js} +1 -1
- package/dist/gateway/static/root/assets/{agent-avatar-dicebear-Bj3rvcTm.js → agent-avatar-dicebear-Stj17S9u.js} +1 -1
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-CCRfvuov.js +1 -0
- package/dist/gateway/static/root/assets/{agent-plugin-dialog-BjdgKCnk.js → agent-plugin-dialog-CRlzd8El.js} +1 -1
- package/dist/gateway/static/root/assets/agents-_ug2LCQb.js +66 -0
- package/dist/gateway/static/root/assets/{agents-admin-api-B5QDepbW.js → agents-admin-api-HBZvVyzF.js} +1 -1
- package/dist/gateway/static/root/assets/{app-management-settings-panel-2rTEtB_X.js → app-management-settings-panel-pJFe6M6c.js} +1 -1
- package/dist/gateway/static/root/assets/{appearance-settings-BPVBS80k.js → appearance-settings-BrxSpBWa.js} +1 -1
- package/dist/gateway/static/root/assets/{apps-page-1FuKlZYP.js → apps-page-DegSl5Lh.js} +1 -1
- package/dist/gateway/static/root/assets/{archive-plugin-BbT1O7oK.js → archive-plugin-BO3I9zRP.js} +1 -1
- package/dist/gateway/static/root/assets/{automations-page-DjwP_h8h.js → automations-page-BZQ_VHjp.js} +1 -1
- package/dist/gateway/static/root/assets/automations-workspace-BQywN6nG.js +6 -0
- package/dist/gateway/static/root/assets/{binary-plugins-gS7Wg68v.js → binary-plugins-gFD7vH9R.js} +1 -1
- package/dist/gateway/static/root/assets/{block-editor-DbwMaivP.js → block-editor-3XjFv3kK.js} +3 -3
- package/dist/gateway/static/root/assets/browser-automation-inputs-DxRoKFun.js +1 -0
- package/dist/gateway/static/root/assets/browser-automations-page-yiEuG1v7.js +3 -0
- package/dist/gateway/static/root/assets/browser-settings-page-ch2_bQF0.js +3 -0
- package/dist/gateway/static/root/assets/{capabilities-Dd1PZRQw.js → capabilities-B48r0DDa.js} +1 -1
- package/dist/gateway/static/root/assets/{capabilities-Lswa0hv8.js → capabilities-DO-vfNKI.js} +1 -1
- package/dist/gateway/static/root/assets/{capabilities-page-djBsC_tG.js → capabilities-page-BEnFt_dj.js} +2 -2
- package/dist/gateway/static/root/assets/{capabilities-settings-panel-DTHR1DrJ.js → capabilities-settings-panel-NckMsJ5N.js} +4 -4
- package/dist/gateway/static/root/assets/{capability-header-actions-494dnJsj.js → capability-header-actions-CY8KCO9j.js} +1 -1
- package/dist/gateway/static/root/assets/channels-CHOS2CGa.js +1 -0
- package/dist/gateway/static/root/assets/{chat-terminal-dock-Bl7-Zk-x.js → chat-terminal-dock-Bwknnb4c.js} +2 -2
- package/dist/gateway/static/root/assets/{computer-settings-page-ByExKr_6.js → computer-settings-page-B2xeFCSt.js} +1 -1
- package/dist/gateway/static/root/assets/{connector-service-page-BK_5aFxO.js → connector-service-page-C83DnAqK.js} +1 -1
- package/dist/gateway/static/root/assets/connectors-page-D6wTmWA6.js +2 -0
- package/dist/gateway/static/root/assets/{date-picker-BXJv6472.js → date-picker-Bq9GR-DU.js} +1 -1
- package/dist/gateway/static/root/assets/{dependency-picker-CWbbhKZa.js → dependency-picker-BTERkA9X.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-DWNEC3mk.js → desktop-pet-BwsbQw8V.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-settings-BYwlxGiN.js → desktop-pet-settings-sL9i44pP.js} +1 -1
- package/dist/gateway/static/root/assets/device-pairing-wizard-DI2bqaKO.js +1 -0
- package/dist/gateway/static/root/assets/{directory-picker-path-field-D26tSY8Z.js → directory-picker-path-field-BFlDmsS5.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-debug-page-xYIGNegB.js → extension-debug-page-7Z1Tp7-w.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-page-BoQ1-3AW.js → extension-page-uGqvb8QX.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-provider-Cw_JU6RR.js → extension-provider-xlSyz4Z6.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-settings-page-BH5TFGPe.js → extension-settings-page-CKHgrjOs.js} +1 -1
- package/dist/gateway/static/root/assets/{fallback-models-04Wqmw14.js → fallback-models-BwQg0G1D.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-swr-CNf0ZeRw.js → gateway-config-swr-7RKqgU_U.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-realtime-C4KtwMY3.js → gateway-realtime-DDWymi-o.js} +1 -1
- package/dist/gateway/static/root/assets/gateway-realtime-store-BheKaO6d.js +1 -0
- package/dist/gateway/static/root/assets/{gateway-settings-BqvCJVUC.js → gateway-settings-BAQnSEMJ.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-startup-retry-DIxHacUw.js → gateway-startup-retry-jiV5sbEX.js} +1 -1
- package/dist/gateway/static/root/assets/home-page-BZDjvwet.js +2 -0
- package/dist/gateway/static/root/assets/{image-generation-api-CS0q0Kto.js → image-generation-api-DO4mXb6I.js} +1 -1
- package/dist/gateway/static/root/assets/{imports-page-n_rSLo6b.js → imports-page-re1_Nh64.js} +1 -1
- package/dist/gateway/static/root/assets/index-BB1z8QxA.js +106 -0
- package/dist/gateway/static/root/assets/index-D9R6tU7r.css +1 -0
- package/dist/gateway/static/root/assets/{keyboard-shortcuts-settings-tVkrHbYF.js → keyboard-shortcuts-settings-B_xja9LN.js} +1 -1
- package/dist/gateway/static/root/assets/{local-app-workbench-page-L8tYuas0.js → local-app-workbench-page-CErdCrPX.js} +1 -1
- package/dist/gateway/static/root/assets/{local-apps-page-SSJ9B1bH.js → local-apps-page-BPL2RXdK.js} +1 -1
- package/dist/gateway/static/root/assets/{local-session-drafts-B_McurIj.js → local-session-drafts-DRPsDCj1.js} +1 -1
- package/dist/gateway/static/root/assets/{locale-store-D6_rF-qT.js → locale-store-COVscAqC.js} +1 -1
- package/dist/gateway/static/root/assets/{logs-page-BWD63eg1.js → logs-page-DL-BM0xq.js} +2 -2
- package/dist/gateway/static/root/assets/management-settings-BOVRHhXk.js +1 -0
- package/dist/gateway/static/root/assets/{markdown-view-_xwUuKce.js → markdown-view-B_qnAcW2.js} +1 -1
- package/dist/gateway/static/root/assets/{media-plugins-C631CDu4.js → media-plugins-C0Pc-wSY.js} +1 -1
- package/dist/gateway/static/root/assets/messages-Bb2GK-Nj.js +3 -0
- package/dist/gateway/static/root/assets/models-hub-cache-DyPPksMP.js +1 -0
- package/dist/gateway/static/root/assets/note-detail-panel-DJJK2yL4.js +5 -0
- package/dist/gateway/static/root/assets/{notes-api-vIZWQ9Yt.js → notes-api-sBgBcyLW.js} +1 -1
- package/dist/gateway/static/root/assets/notes-page-oCxO7ZTv.js +8 -0
- package/dist/gateway/static/root/assets/{page-context-capture-button-BlPDM7GM.js → page-context-capture-button-tf77xIz3.js} +1 -1
- package/dist/gateway/static/root/assets/personal-page-DOk9gUK5.js +1 -0
- package/dist/gateway/static/root/assets/{product-open-page-BJjWvoER.js → product-open-page-Dc3Y-Y7J.js} +1 -1
- package/dist/gateway/static/root/assets/{product-preview-content-CEd4E7Gf.js → product-preview-content-MG_hmVA3.js} +1 -1
- package/dist/gateway/static/root/assets/{project-detail-page-DM4_VLNX.js → project-detail-page-3uAzbW4T.js} +1 -1
- package/dist/gateway/static/root/assets/{projects-page-lszV8HuS.js → projects-page-MllfNLGA.js} +1 -1
- package/dist/gateway/static/root/assets/{registry-api-uJnjqTcn.js → registry-api-Bh9UBb-f.js} +1 -1
- package/dist/gateway/static/root/assets/{remote-access-hub-O6j9CWX0.js → remote-access-hub-Bs-C70el.js} +1 -1
- package/dist/gateway/static/root/assets/{runtime-tools-settings-panel-CjnNQYW8.js → runtime-tools-settings-panel-DpJ5jNzR.js} +1 -1
- package/dist/gateway/static/root/assets/{scenes-page-B3T3dDOR.js → scenes-page-Dpxamkw6.js} +2 -2
- package/dist/gateway/static/root/assets/{schema-form-DT3c4Vd_.js → schema-form-DUeyQ1zh.js} +1 -1
- package/dist/gateway/static/root/assets/{send-session-input-CICkGwn-.js → send-session-input-Dgn4BRk_.js} +1 -1
- package/dist/gateway/static/root/assets/{session-api-BcQodoAu.js → session-api-yjpEz4pP.js} +1 -1
- package/dist/gateway/static/root/assets/{session-manager-B0tOE4qz.js → session-manager-LInoZ_Yi.js} +1 -1
- package/dist/gateway/static/root/assets/sessions-page-BCVlNHtq.js +1 -0
- package/dist/gateway/static/root/assets/settings-advanced-gate-Bz-ue9fZ.js +1 -0
- package/dist/gateway/static/root/assets/{settings-page-D-PZjcss.js → settings-page-DouHuBXt.js} +1 -1
- package/dist/gateway/static/root/assets/{setup-status-panel-Ce3gTbT3.js → setup-status-panel-B5BoZ-1y.js} +1 -1
- package/dist/gateway/static/root/assets/{share-link-dialog-CWS8EkJS.js → share-link-dialog-DJTabLAM.js} +1 -1
- package/dist/gateway/static/root/assets/{share-preview-page-23NJAKiq.js → share-preview-page-jw7h4t_B.js} +1 -1
- package/dist/gateway/static/root/assets/{shares-api-iKhHbvBD.js → shares-api-c0dfWz6h.js} +1 -1
- package/dist/gateway/static/root/assets/shares-settings-BlnlzGjH.js +1 -0
- package/dist/gateway/static/root/assets/{skill-api-C-MWdQwl.js → skill-api-CfyX_0zN.js} +1 -1
- package/dist/gateway/static/root/assets/{skill-reload-api-iy02aB13.js → skill-reload-api-Ge5XymHa.js} +1 -1
- package/dist/gateway/static/root/assets/skills-page-DjdTeYzm.js +2 -0
- package/dist/gateway/static/root/assets/system-settings-panel-DWf256TE.js +1 -0
- package/dist/gateway/static/root/assets/{task-detail-page-B2SIEyhZ.js → task-detail-page-DE6N1ZQe.js} +2 -2
- package/dist/gateway/static/root/assets/{task-expandable-content-D2LRR4BC.js → task-expandable-content-hYcgCEE3.js} +1 -1
- package/dist/gateway/static/root/assets/{theme-store-BJfhz-Zq.js → theme-store-BOfCKgl6.js} +1 -1
- package/dist/gateway/static/root/assets/{tracing-settings-page-aHD56iaS.js → tracing-settings-page-DDWFXcST.js} +1 -1
- package/dist/gateway/static/root/assets/{tunnel-api-BCf8CfTC.js → tunnel-api-BBm9esrY.js} +1 -1
- package/dist/gateway/static/root/assets/{usage-page-B4XUkuJd.js → usage-page-C93x89uW.js} +1 -1
- package/dist/gateway/static/root/assets/{use-autosave-BnnTw9RE.js → use-autosave-CSckcjFx.js} +1 -1
- package/dist/gateway/static/root/assets/{use-composer-attachments-Itczg0y1.js → use-composer-attachments-C7Gaaai1.js} +1 -1
- package/dist/gateway/static/root/assets/{use-file-preview-expanded-DJ5vyrvM.js → use-file-preview-expanded-Db2VYg1b.js} +2 -2
- package/dist/gateway/static/root/assets/user-model-page-B74LkVbp.js +2 -0
- package/dist/gateway/static/root/assets/{voice-api-key-field-BphJku8x.js → voice-api-key-field-GEw5x2fU.js} +1 -1
- package/dist/gateway/static/root/assets/{voice-config-api-B4u4coFj.js → voice-config-api-3yyVwXln.js} +1 -1
- package/dist/gateway/static/root/assets/{work-discovery-overlay-jgOEP1xw.js → work-discovery-overlay-Bs2MKiTv.js} +1 -1
- package/dist/gateway/static/root/assets/{workflow-run-setup-panel-N69crzRu.js → workflow-run-setup-panel-BR_U_snV.js} +2 -2
- package/dist/gateway/static/root/assets/workflows-page-Crk4SxtC.js +6 -0
- package/dist/gateway/static/root/assets/{workspace-api-Lzv6GDO8.js → workspace-api-Si2o3inw.js} +2 -2
- package/dist/gateway/static/root/index.html +27 -27
- package/dist/package.js +1 -1
- package/dist/packages/browser-control-contract/src/automation.js +213 -0
- package/dist/packages/browser-control-contract/src/index.js +43 -12
- package/dist/packages/gateway-contract/src/gateway-scopes.js +3 -1
- package/dist/packages/gateway-contract/src/notifications.js +1 -0
- package/dist/packages/realtime-protocol/src/voice.js +9 -3
- package/dist/src/agent/coding/repository-instructions.js +8 -1
- package/dist/src/agent/tool-manuals/browser.d.ts +1 -1
- package/dist/src/agent/tool-manuals/browser.js +3 -3
- package/dist/src/agent/tools/browser/tool/browser-use-tool.js +1 -1
- package/dist/src/agent/tools/browser/tool/schemas.d.ts +0 -13
- package/dist/src/agent/tools/browser/tool/schemas.js +0 -4
- package/dist/src/agent/tools/browser-automation-tool.d.ts +74 -1
- package/dist/src/agent/tools/browser-automation-tool.js +52 -6
- package/dist/src/automations/capabilities/write.d.ts +2 -0
- package/dist/src/automations/domain/types.d.ts +11 -0
- package/dist/src/automations/domain/validation.d.ts +4 -0
- package/dist/src/automations/domain/validation.js +1 -0
- package/dist/src/automations/draft/automation-draft-validator.d.ts +1 -0
- package/dist/src/automations/service/action-executor.js +11 -3
- package/dist/src/automations/service/automation-service.d.ts +1 -0
- package/dist/src/automations/service/automation-service.js +13 -1
- package/dist/src/bootstrap/application-state.js +1 -1
- package/dist/src/browser/automations/repository.d.ts +4 -0
- package/dist/src/browser/automations/repository.js +27 -6
- package/dist/src/browser/automations/runner.js +101 -9
- package/dist/src/browser/automations/runtime-service.js +4 -3
- package/dist/src/browser/automations/schema.d.ts +84 -1
- package/dist/src/browser/automations/schema.js +68 -2
- package/dist/src/browser/automations/service.d.ts +78 -1
- package/dist/src/browser/automations/service.js +36 -4
- package/dist/src/browser/automations/types.d.ts +19 -4
- package/dist/src/browser/drivers/extension-driver.d.ts +5 -5
- package/dist/src/browser/drivers/extension-driver.js +10 -10
- package/dist/src/browser/drivers/playwright-driver.js +46 -0
- package/dist/src/browser/index.d.ts +1 -2
- package/dist/src/browser/index.js +2 -3
- package/dist/src/browser/policy/browser-policy.d.ts +0 -1
- package/dist/src/browser/policy/browser-policy.js +1 -4
- package/dist/src/browser/providers/index.d.ts +1 -1
- package/dist/src/browser/providers/index.js +2 -2
- package/dist/src/browser/providers/realtime-extension.d.ts +1 -1
- package/dist/src/browser/providers/realtime-extension.js +5 -1
- package/dist/src/browser/recordings/service.d.ts +22 -0
- package/dist/src/browser/recordings/service.js +160 -0
- package/dist/src/browser/runtime/browser-runtime.d.ts +1 -1
- package/dist/src/browser/runtime/browser-runtime.js +19 -56
- package/dist/src/cli/commands/doctor/checks/database-schema.js +1 -1
- package/dist/src/computer/runtime.js +2 -2
- package/dist/src/config/schema.d.ts +12 -12
- package/dist/src/config/schema.js +12 -12
- package/dist/src/endpoint-tools/policy.js +4 -3
- package/dist/src/gateway/hono/routes/browser-automations.js +23 -7
- package/dist/src/gateway/hono/routes/browser-recordings.d.ts +3 -0
- package/dist/src/gateway/hono/routes/browser-recordings.js +78 -0
- package/dist/src/gateway/hono/routes/browser.js +1 -34
- package/dist/src/gateway/hono/routes/device-push.js +1 -0
- package/dist/src/gateway/hono/routes/home.js +1 -1
- package/dist/src/gateway/hono/routes/lazy-bundles.js +8 -0
- package/dist/src/gateway/hono/routes/personal-agent.js +35 -0
- package/dist/src/gateway/hono/routes/voice.js +5 -2
- package/dist/src/gateway/security/gateway-scopes.js +3 -1
- package/dist/src/gateway/service/agent-runner.js +1 -0
- package/dist/src/home-intelligence/generator.d.ts +2 -0
- package/dist/src/home-intelligence/generator.js +53 -4
- package/dist/src/home-intelligence/host.d.ts +2 -1
- package/dist/src/home-intelligence/host.js +23 -3
- package/dist/src/migrations/cloud-public-models.js +1 -0
- package/dist/src/notifications/clarification.d.ts +5 -0
- package/dist/src/notifications/clarification.js +66 -0
- package/dist/src/notifications/device-store.js +1 -0
- package/dist/src/notifications/harmony-push.d.ts +1 -0
- package/dist/src/notifications/harmony-push.js +3 -1
- package/dist/src/notifications/planner.d.ts +1 -0
- package/dist/src/notifications/service.d.ts +2 -2
- package/dist/src/notifications/service.js +20 -4
- package/dist/src/notifications/store.d.ts +2 -0
- package/dist/src/notifications/store.js +7 -1
- package/dist/src/notifications/types.d.ts +1 -0
- package/dist/src/notifications/types.js +1 -0
- package/dist/src/storage/sqlite/clarification-wait-repository.js +1 -0
- package/dist/src/storage/sqlite/migrations/238_browser_record_replay.sql +45 -0
- package/dist/src/storage/sqlite/migrations/239_clarification_notifications.sql +26 -0
- package/dist/src/storage/sqlite/migrations/runner.d.ts +1 -1
- package/dist/src/storage/sqlite/migrations/runner.js +3 -3
- package/dist/src/storage/sqlite/schema.js +1 -1
- package/dist/src/tasks/home-query-service.d.ts +1 -1
- package/dist/src/tasks/home-query-service.js +6 -2
- package/dist/src/voice/dashscope/streaming-tts-stream.d.ts +2 -1
- package/dist/src/voice/dashscope/streaming-tts-stream.js +135 -79
- package/dist/src/voice/realtime/agentEngine.js +78 -1
- package/dist/src/voice/realtime/speakable-segmenter.d.ts +2 -0
- package/dist/src/voice/realtime/speakable-segmenter.js +30 -0
- package/dist/src/voice/tts/providers/alibaba-speech.js +17 -1
- package/dist/src/voice/tts/speak-core.d.ts +3 -0
- package/dist/src/voice/tts/speak-core.js +21 -1
- package/dist/src/voice/tts/speech-provider-types.d.ts +8 -0
- package/package.json +3 -3
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-D0tD3kCe.js +0 -1
- package/dist/gateway/static/root/assets/agents-CWXKzX5L.js +0 -66
- package/dist/gateway/static/root/assets/automations-workspace-C9ZT6TAU.js +0 -6
- package/dist/gateway/static/root/assets/browser-automation-inputs-CXG_JveC.js +0 -1
- package/dist/gateway/static/root/assets/browser-automations-page-CJc7OKR3.js +0 -3
- package/dist/gateway/static/root/assets/browser-settings-page-BjaTEUR8.js +0 -3
- package/dist/gateway/static/root/assets/channels-BaYRJtz3.js +0 -1
- package/dist/gateway/static/root/assets/connectors-page-DyqRwSH2.js +0 -2
- package/dist/gateway/static/root/assets/device-pairing-wizard-BAc4F35b.js +0 -1
- package/dist/gateway/static/root/assets/gateway-realtime-store-ChZUzVbe.js +0 -1
- package/dist/gateway/static/root/assets/home-page-DhPPLbdD.js +0 -2
- package/dist/gateway/static/root/assets/index-CK4O2xMa.css +0 -1
- package/dist/gateway/static/root/assets/index-DpLD-7Hc.js +0 -106
- package/dist/gateway/static/root/assets/management-settings-aKSibklM.js +0 -1
- package/dist/gateway/static/root/assets/messages-Bnx3GMDB.js +0 -3
- package/dist/gateway/static/root/assets/models-hub-cache-Di83uIPw.js +0 -1
- package/dist/gateway/static/root/assets/note-detail-panel-B_fve3_2.js +0 -5
- package/dist/gateway/static/root/assets/notes-page-DnChyp8S.js +0 -8
- package/dist/gateway/static/root/assets/personal-page-BM2qAgg_.js +0 -1
- package/dist/gateway/static/root/assets/sessions-page-BIixHXlQ.js +0 -1
- package/dist/gateway/static/root/assets/settings-advanced-gate-49SUEWPh.js +0 -1
- package/dist/gateway/static/root/assets/shares-settings-CNEZjF66.js +0 -1
- package/dist/gateway/static/root/assets/skills-page-BndNdYsJ.js +0 -2
- package/dist/gateway/static/root/assets/system-settings-panel-EpZeMlvp.js +0 -1
- package/dist/gateway/static/root/assets/user-model-page-Bz5lkFG9.js +0 -2
- package/dist/gateway/static/root/assets/workflows-page-CPCgOfHh.js +0 -6
- package/dist/src/browser/policy/approval-store.d.ts +0 -17
- package/dist/src/browser/policy/approval-store.js +0 -62
- package/dist/src/browser/providers/extension-ws-acquire.d.ts +0 -19
- package/dist/src/browser/providers/extension-ws-acquire.js +0 -78
- package/dist/src/browser/providers/extension.d.ts +0 -61
- package/dist/src/browser/providers/extension.js +0 -378
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
//#region packages/browser-control-contract/src/automation.ts
|
|
2
|
+
/** Runs in the page. Keep all helpers inside this function for serialization. */
|
|
3
|
+
async function browserDom(input) {
|
|
4
|
+
const root = globalThis;
|
|
5
|
+
const doc = root.document;
|
|
6
|
+
const normalize = (value) => String(value || "").trim().replace(/\s+/g, " ").slice(0, 240);
|
|
7
|
+
const sensitive = (element) => element.type === "password" || element.type === "file" || /cc-|one-time-code|transaction-|password|token|secret|otp|cvv|cvc|card.?number|验证码|密码|银行卡|卡号/i.test([
|
|
8
|
+
element.autocomplete,
|
|
9
|
+
element.name,
|
|
10
|
+
element.id,
|
|
11
|
+
element.getAttribute("aria-label"),
|
|
12
|
+
name(element)
|
|
13
|
+
].join(" "));
|
|
14
|
+
const role = (element) => element.getAttribute("role") || {
|
|
15
|
+
A: "link",
|
|
16
|
+
BUTTON: "button",
|
|
17
|
+
SUMMARY: "button",
|
|
18
|
+
SELECT: "combobox",
|
|
19
|
+
TEXTAREA: "textbox",
|
|
20
|
+
FORM: "form",
|
|
21
|
+
FIELDSET: "group",
|
|
22
|
+
SECTION: "region"
|
|
23
|
+
}[element.tagName] || (element.tagName === "INPUT" ? element.type === "checkbox" ? "checkbox" : element.type === "radio" ? "radio" : [
|
|
24
|
+
"submit",
|
|
25
|
+
"button",
|
|
26
|
+
"reset"
|
|
27
|
+
].includes(element.type) ? "button" : "textbox" : "text");
|
|
28
|
+
const name = (element) => normalize((element.getAttribute("aria-labelledby") || "").split(/\s+/).map((id) => doc.getElementById(id)?.textContent || "").join(" ") || element.getAttribute("aria-label") || element.labels?.[0]?.textContent || (element.tagName === "FIELDSET" ? element.querySelector("legend")?.textContent : "") || element.getAttribute("alt") || element.getAttribute("placeholder") || ([
|
|
29
|
+
"submit",
|
|
30
|
+
"button",
|
|
31
|
+
"reset"
|
|
32
|
+
].includes(element.type) ? element.value : "") || element.innerText || element.getAttribute("title") || "");
|
|
33
|
+
const describe = (element) => {
|
|
34
|
+
const container = element.parentElement?.closest("form,fieldset,section,[role=\"region\"],[role=\"group\"],[role=\"dialog\"]");
|
|
35
|
+
return {
|
|
36
|
+
role: role(element),
|
|
37
|
+
name: name(element),
|
|
38
|
+
...element.getAttribute("data-testid") ? { testId: element.getAttribute("data-testid") } : {},
|
|
39
|
+
...container && name(container) ? { scope: {
|
|
40
|
+
role: role(container),
|
|
41
|
+
name: name(container)
|
|
42
|
+
} } : {}
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
const selector = "a[href],button,input:not([type=\"hidden\"]),textarea,select,summary,[role],[data-testid],label,p,span,h1,h2,h3,td,output";
|
|
46
|
+
if (input.operation === "resolve") {
|
|
47
|
+
const target = input.target;
|
|
48
|
+
const matches = Array.from(doc.querySelectorAll(selector)).filter((element) => {
|
|
49
|
+
const rect = element.getBoundingClientRect();
|
|
50
|
+
const style = root.getComputedStyle(element);
|
|
51
|
+
if (!rect.width || !rect.height || style.display === "none" || style.visibility === "hidden") return false;
|
|
52
|
+
const descriptor = describe(element);
|
|
53
|
+
return descriptor.role === target.role && (!target.name || descriptor.name === target.name) && (!target.nameIncludes || descriptor.name?.includes(target.nameIncludes)) && (!target.testId || descriptor.testId === target.testId) && (!target.scope || descriptor.scope?.role === target.scope.role && descriptor.scope?.name === target.scope.name);
|
|
54
|
+
});
|
|
55
|
+
if (matches.length !== 1) return { matches: matches.length };
|
|
56
|
+
const element = matches[0];
|
|
57
|
+
const ref = `xopc-e${++root.__xopcRefCounter || (root.__xopcRefCounter = 1)}`;
|
|
58
|
+
element.setAttribute("data-xopc-ref", ref);
|
|
59
|
+
return {
|
|
60
|
+
matches: 1,
|
|
61
|
+
node: {
|
|
62
|
+
ref,
|
|
63
|
+
role: role(element),
|
|
64
|
+
name: name(element),
|
|
65
|
+
value: !sensitive(element) && "value" in element ? String(element.value).slice(0, 4096) : void 0,
|
|
66
|
+
description: !sensitive(element) ? normalize(element.textContent) : void 0,
|
|
67
|
+
href: element.tagName === "A" ? element.href : void 0,
|
|
68
|
+
states: [
|
|
69
|
+
sensitive(element) ? "sensitive" : "",
|
|
70
|
+
element.disabled ? "disabled" : "",
|
|
71
|
+
element.checked ? "checked" : "",
|
|
72
|
+
["BUTTON", "INPUT"].includes(element.tagName) && element.type === "submit" ? "submit" : ""
|
|
73
|
+
].filter(Boolean)
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
const prior = root.__xopcRecorder;
|
|
78
|
+
if (input.operation === "record" && prior?.recordingId === input.recordingId && prior?.generation === input.generation) return { recording: true };
|
|
79
|
+
if (prior) {
|
|
80
|
+
await prior.stop();
|
|
81
|
+
delete root.__xopcRecorder;
|
|
82
|
+
}
|
|
83
|
+
if (input.operation === "stop") return { stopped: true };
|
|
84
|
+
let sourceSeq = 0;
|
|
85
|
+
const pending = /* @__PURE__ */ new Set();
|
|
86
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
87
|
+
const lastValues = /* @__PURE__ */ new WeakMap();
|
|
88
|
+
const listeners = [];
|
|
89
|
+
const cleanUrl = () => {
|
|
90
|
+
const url = new URL(root.location.href);
|
|
91
|
+
url.username = "";
|
|
92
|
+
url.password = "";
|
|
93
|
+
url.hash = "";
|
|
94
|
+
for (const key of [...url.searchParams.keys()]) if (/token|secret|password|key|code/i.test(key)) url.searchParams.delete(key);
|
|
95
|
+
return url.toString();
|
|
96
|
+
};
|
|
97
|
+
const documentId = `${root.performance.timeOrigin}:${input.generation}`;
|
|
98
|
+
let failed = false;
|
|
99
|
+
let ended = false;
|
|
100
|
+
let enterTarget;
|
|
101
|
+
let enterAt = 0;
|
|
102
|
+
const emit = (event) => {
|
|
103
|
+
const task = root.chrome.runtime.sendMessage({
|
|
104
|
+
type: "browser/recording-event",
|
|
105
|
+
recordingId: input.recordingId,
|
|
106
|
+
generation: input.generation,
|
|
107
|
+
event: {
|
|
108
|
+
...event,
|
|
109
|
+
id: root.crypto.randomUUID(),
|
|
110
|
+
sourceSeq: ++sourceSeq,
|
|
111
|
+
documentId,
|
|
112
|
+
url: cleanUrl()
|
|
113
|
+
}
|
|
114
|
+
}).then((ack) => {
|
|
115
|
+
if (!ack?.ok) failed = true;
|
|
116
|
+
}).catch(() => {
|
|
117
|
+
failed = true;
|
|
118
|
+
});
|
|
119
|
+
pending.add(task);
|
|
120
|
+
task.finally(() => pending.delete(task));
|
|
121
|
+
};
|
|
122
|
+
const flush = (element) => {
|
|
123
|
+
if (!dirty.delete(element) || sensitive(element)) return;
|
|
124
|
+
const value = String(element.value || "").slice(0, 4096);
|
|
125
|
+
if (lastValues.get(element) === value) return;
|
|
126
|
+
lastValues.set(element, value);
|
|
127
|
+
emit({
|
|
128
|
+
action: element.tagName === "SELECT" ? "select" : "fill",
|
|
129
|
+
target: describe(element),
|
|
130
|
+
value
|
|
131
|
+
});
|
|
132
|
+
};
|
|
133
|
+
const listen = (type, handler) => {
|
|
134
|
+
listeners.push([type, handler]);
|
|
135
|
+
(type === "pagehide" ? root : doc).addEventListener(type, handler, true);
|
|
136
|
+
};
|
|
137
|
+
listen("input", (event) => {
|
|
138
|
+
if (event.isTrusted && !sensitive(event.target) && !["checkbox", "radio"].includes(event.target.type) && "value" in event.target) dirty.add(event.target);
|
|
139
|
+
});
|
|
140
|
+
listen("dragstart", (event) => {
|
|
141
|
+
if (event.isTrusted) emit({
|
|
142
|
+
action: "unsupported",
|
|
143
|
+
value: "Drag actions are not supported by the recorder."
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
listen("change", (event) => {
|
|
147
|
+
if (!event.isTrusted || sensitive(event.target)) return;
|
|
148
|
+
if (["checkbox", "radio"].includes(event.target.type)) emit({
|
|
149
|
+
action: "check",
|
|
150
|
+
target: describe(event.target),
|
|
151
|
+
checked: !!event.target.checked
|
|
152
|
+
});
|
|
153
|
+
else {
|
|
154
|
+
dirty.add(event.target);
|
|
155
|
+
flush(event.target);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
listen("focusout", (event) => flush(event.target));
|
|
159
|
+
listen("click", (event) => {
|
|
160
|
+
if (!event.isTrusted) return;
|
|
161
|
+
for (const element of dirty) flush(element);
|
|
162
|
+
const element = event.target.closest("a[href],button,summary,[role=\"button\"],input[type=\"submit\"],input[type=\"button\"]");
|
|
163
|
+
if (element && event.detail === 0 && Date.now() - enterAt < 500 && (element === enterTarget || element.form && element.form === enterTarget?.form)) return;
|
|
164
|
+
if (element && !sensitive(element)) emit({
|
|
165
|
+
action: "click",
|
|
166
|
+
target: describe(element)
|
|
167
|
+
});
|
|
168
|
+
else if (event.target.closest("canvas,[contenteditable=\"true\"],input[type=\"file\"]")) emit({
|
|
169
|
+
action: "unsupported",
|
|
170
|
+
value: "Canvas, rich text and file actions require an assistant to complete the workflow."
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
listen("pagehide", () => {
|
|
174
|
+
for (const element of dirty) flush(element);
|
|
175
|
+
emit({ action: "checkpoint" });
|
|
176
|
+
ended = true;
|
|
177
|
+
for (const [type, handler] of listeners) (type === "pagehide" ? root : doc).removeEventListener(type, handler, true);
|
|
178
|
+
});
|
|
179
|
+
listen("keydown", (event) => {
|
|
180
|
+
if (!event.isTrusted || event.isComposing || sensitive(event.target) || !["Enter", "Escape"].includes(event.key)) return;
|
|
181
|
+
flush(event.target);
|
|
182
|
+
if (event.key === "Enter") {
|
|
183
|
+
enterTarget = event.target;
|
|
184
|
+
enterAt = Date.now();
|
|
185
|
+
}
|
|
186
|
+
emit({
|
|
187
|
+
action: "press",
|
|
188
|
+
target: describe(event.target),
|
|
189
|
+
key: event.key
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
root.__xopcRecorder = {
|
|
193
|
+
recordingId: input.recordingId,
|
|
194
|
+
generation: input.generation,
|
|
195
|
+
stop: async () => {
|
|
196
|
+
for (const [type, handler] of listeners) (type === "pagehide" ? root : doc).removeEventListener(type, handler, true);
|
|
197
|
+
for (const element of dirty) flush(element);
|
|
198
|
+
if (!ended) {
|
|
199
|
+
emit({ action: "checkpoint" });
|
|
200
|
+
ended = true;
|
|
201
|
+
}
|
|
202
|
+
await Promise.all([...pending]);
|
|
203
|
+
if (failed) throw new Error("Recording events could not be saved.");
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
emit({ action: "navigate" });
|
|
207
|
+
return {
|
|
208
|
+
recording: true,
|
|
209
|
+
documentId
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
//#endregion
|
|
213
|
+
export { browserDom };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import "./automation.js";
|
|
1
2
|
//#region packages/browser-control-contract/src/index.ts
|
|
2
3
|
const BROWSER_CONTROL_ENDPOINT_TOOL_NAME = "browser.control";
|
|
3
4
|
const BROWSER_CONTROL_ENDPOINT_INPUT_SCHEMA = {
|
|
@@ -28,16 +29,46 @@ const BROWSER_CONTROL_ENDPOINT_OUTPUT_SCHEMA = {
|
|
|
28
29
|
}
|
|
29
30
|
}
|
|
30
31
|
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
32
|
+
const BROWSER_CONTROL_ENDPOINT_DESCRIPTOR = {
|
|
33
|
+
name: BROWSER_CONTROL_ENDPOINT_TOOL_NAME,
|
|
34
|
+
title: "Browser control transport",
|
|
35
|
+
description: "Internal authenticated transport for xopc browser-control actions.",
|
|
36
|
+
inputSchema: BROWSER_CONTROL_ENDPOINT_INPUT_SCHEMA,
|
|
37
|
+
outputSchema: BROWSER_CONTROL_ENDPOINT_OUTPUT_SCHEMA,
|
|
38
|
+
policyId: "browser.control",
|
|
39
|
+
sensitivity: "personal",
|
|
40
|
+
effect: "write",
|
|
41
|
+
confirmation: "never",
|
|
42
|
+
requiresForeground: false,
|
|
43
|
+
requiredPermissions: ["browser-control"],
|
|
44
|
+
timeoutMs: 12e4,
|
|
45
|
+
maxConcurrency: 4,
|
|
46
|
+
supportsCancellation: true,
|
|
47
|
+
idempotent: false,
|
|
48
|
+
resultKinds: ["json"]
|
|
49
|
+
};
|
|
50
|
+
const BROWSER_RECORDING_ENDPOINT_INPUT_SCHEMA = {
|
|
51
|
+
type: "object",
|
|
52
|
+
additionalProperties: false,
|
|
53
|
+
required: ["operation"],
|
|
54
|
+
properties: { operation: {
|
|
55
|
+
type: "string",
|
|
56
|
+
enum: [
|
|
57
|
+
"start",
|
|
58
|
+
"pause",
|
|
59
|
+
"resume",
|
|
60
|
+
"finish",
|
|
61
|
+
"status"
|
|
62
|
+
]
|
|
63
|
+
} }
|
|
64
|
+
};
|
|
65
|
+
const BROWSER_RECORDING_ENDPOINT_DESCRIPTOR = {
|
|
66
|
+
...BROWSER_CONTROL_ENDPOINT_DESCRIPTOR,
|
|
67
|
+
name: "browser.recording",
|
|
68
|
+
title: "Browser workflow recording",
|
|
69
|
+
description: "Record a workflow in the active browser tab and save it as an automation.",
|
|
70
|
+
inputSchema: BROWSER_RECORDING_ENDPOINT_INPUT_SCHEMA,
|
|
71
|
+
maxConcurrency: 1
|
|
72
|
+
};
|
|
42
73
|
//#endregion
|
|
43
|
-
export { BROWSER_CONTROL_ENDPOINT_INPUT_SCHEMA, BROWSER_CONTROL_ENDPOINT_OUTPUT_SCHEMA, BROWSER_CONTROL_ENDPOINT_TOOL_NAME,
|
|
74
|
+
export { BROWSER_CONTROL_ENDPOINT_DESCRIPTOR, BROWSER_CONTROL_ENDPOINT_INPUT_SCHEMA, BROWSER_CONTROL_ENDPOINT_OUTPUT_SCHEMA, BROWSER_CONTROL_ENDPOINT_TOOL_NAME, BROWSER_RECORDING_ENDPOINT_DESCRIPTOR, BROWSER_RECORDING_ENDPOINT_INPUT_SCHEMA };
|
|
@@ -10,7 +10,7 @@ function parseVoiceClientJsonFrame(text) {
|
|
|
10
10
|
if (new TextEncoder().encode(text).byteLength > 16 * 1024) throw new Error("Voice control frame exceeds 16384 bytes");
|
|
11
11
|
return JSON.parse(text);
|
|
12
12
|
}
|
|
13
|
-
var VOICE_REALTIME_WS_PATH, VOICE_REALTIME_PROXY_WS_PATH, VOICE_REALTIME_MAX_BINARY_FRAME_BYTES, VOICE_REALTIME_START_TIMEOUT_MS, VOICE_REALTIME_HEARTBEAT_INTERVAL_MS, idSchema, timestampSchema, voicePurposeSchema, voiceModeSchema, voiceInputModeSchema, voiceLanguageSchema, voicePcmFormatSchema, voiceProviderRouteSchema, voiceRouteSchema, availabilitySchema, createVoiceSessionRequestSchema, clientEnvelope, voiceClientMessageSchema, transcriptPayloadSchema, serverEnvelope;
|
|
13
|
+
var VOICE_REALTIME_WS_PATH, VOICE_REALTIME_PROXY_WS_PATH, VOICE_REALTIME_MAX_BINARY_FRAME_BYTES, VOICE_REALTIME_START_TIMEOUT_MS, VOICE_REALTIME_HEARTBEAT_INTERVAL_MS, idSchema, timestampSchema, voicePurposeSchema, voiceModeSchema, voiceInputModeSchema, voiceLanguageSchema, VOICE_TIMING_METRICS, voicePcmFormatSchema, voiceProviderRouteSchema, voiceRouteSchema, availabilitySchema, createVoiceSessionRequestSchema, clientEnvelope, voiceClientMessageSchema, transcriptPayloadSchema, serverEnvelope;
|
|
14
14
|
var init_voice = __esmMin((() => {
|
|
15
15
|
init_voice_catalog();
|
|
16
16
|
init_voice_audio();
|
|
@@ -25,6 +25,12 @@ var init_voice = __esmMin((() => {
|
|
|
25
25
|
voiceModeSchema = z.enum(["natural", "assistant"]);
|
|
26
26
|
voiceInputModeSchema = z.literal("server_vad");
|
|
27
27
|
voiceLanguageSchema = z.enum(["zh", "en"]);
|
|
28
|
+
VOICE_TIMING_METRICS = [
|
|
29
|
+
"speech_end_to_audio_received",
|
|
30
|
+
"speech_end_to_audio_scheduled",
|
|
31
|
+
"speech_end_to_audio_buffered",
|
|
32
|
+
"local_stop"
|
|
33
|
+
];
|
|
28
34
|
voicePcmFormatSchema = z.strictObject({
|
|
29
35
|
encoding: z.literal("pcm_s16le"),
|
|
30
36
|
sampleRate: z.union([z.literal(16e3), z.literal(24e3)]),
|
|
@@ -126,7 +132,7 @@ var init_voice = __esmMin((() => {
|
|
|
126
132
|
})),
|
|
127
133
|
clientEnvelope("session.metric", z.strictObject({
|
|
128
134
|
responseId: z.string().min(1).max(160),
|
|
129
|
-
metric: z.enum(
|
|
135
|
+
metric: z.enum(VOICE_TIMING_METRICS),
|
|
130
136
|
durationMs: z.number().finite().min(0).max(6e5)
|
|
131
137
|
})),
|
|
132
138
|
clientEnvelope("input.mute", z.strictObject({ muted: z.boolean() })),
|
|
@@ -268,4 +274,4 @@ var init_voice = __esmMin((() => {
|
|
|
268
274
|
}));
|
|
269
275
|
//#endregion
|
|
270
276
|
init_voice();
|
|
271
|
-
export { VOICE_REALTIME_HEARTBEAT_INTERVAL_MS, VOICE_REALTIME_MAX_BINARY_FRAME_BYTES, VOICE_REALTIME_PROXY_WS_PATH, VOICE_REALTIME_START_TIMEOUT_MS, VOICE_REALTIME_WS_PATH, createVoiceSessionRequestSchema, init_voice, parseVoiceClientJsonFrame, parseVoiceClientMessage };
|
|
277
|
+
export { VOICE_REALTIME_HEARTBEAT_INTERVAL_MS, VOICE_REALTIME_MAX_BINARY_FRAME_BYTES, VOICE_REALTIME_PROXY_WS_PATH, VOICE_REALTIME_START_TIMEOUT_MS, VOICE_REALTIME_WS_PATH, VOICE_TIMING_METRICS, createVoiceSessionRequestSchema, init_voice, parseVoiceClientJsonFrame, parseVoiceClientMessage };
|
|
@@ -103,7 +103,14 @@ var RepositoryInstructions = class RepositoryInstructions {
|
|
|
103
103
|
try {
|
|
104
104
|
const canonical = await realpath(file);
|
|
105
105
|
if (!inside(this.root, canonical)) throw new Error(`Instruction file escapes repository: ${file}`);
|
|
106
|
-
|
|
106
|
+
const metadata = await stat(canonical);
|
|
107
|
+
if (metadata.size > 32768) {
|
|
108
|
+
const revision = `oversized:${canonical}:${metadata.size}:${metadata.mtimeMs}:${metadata.ctimeMs}`;
|
|
109
|
+
if (this.seen.get(file) === revision) continue;
|
|
110
|
+
this.pending.set(file, revision);
|
|
111
|
+
sections.push(`Instructions for ${dir} and its descendants (${file}):\nThis instruction file exceeds the 32 KiB automatic inclusion limit (${metadata.size} bytes). Read ${file} explicitly with read_file, in successive chunks if needed, until the entire file has been read. Apply its instructions before working in this scope.`);
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
107
114
|
const content = await readFile(canonical, "utf8");
|
|
108
115
|
if (this.seen.get(file) === content) continue;
|
|
109
116
|
this.pending.set(file, content);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const browserUseManual = "# Browser Tool Manual\n\n## browser_use\n\nUse one typed action per call. Start with `observe` or `navigate`. Every observation returns a `sessionId`, `revision`, and semantic element refs. Element actions must use refs from the newest revision; never invent refs.\n\nAvailable actions:\n\n- `observe`: return URL, title, interactive semantic nodes, and changes. Set `visual` to `always` only when visual evidence is needed.\n- `navigate`: open an HTTP(S) URL and optionally verify an expectation.\n- `click`, `fill`, `select`: act on a semantic ref with its observation revision.\n- `press`, `scroll`: act on an optional ref or the active page.\n- `wait`: wait for page idle, text, or a ref visibility state.\n- `upload`: attach workspace files to a file input
|
|
1
|
+
export declare const browserUseManual = "# Browser Tool Manual\n\n## browser_use\n\nUse one typed action per call. Start with `observe` or `navigate`. Every observation returns a `sessionId`, `revision`, and semantic element refs. Element actions must use refs from the newest revision; never invent refs.\n\nAvailable actions:\n\n- `observe`: return URL, title, interactive semantic nodes, and changes. Set `visual` to `always` only when visual evidence is needed.\n- `navigate`: open an HTTP(S) URL and optionally verify an expectation.\n- `click`, `fill`, `select`: act on a semantic ref with its observation revision.\n- `press`, `scroll`: act on an optional ref or the active page.\n- `wait`: wait for page idle, text, or a ref visibility state.\n- `upload`: attach workspace files to a file input.\n- `tabs`: list, create, activate, or close task-owned tabs.\n- `sequence`: run a short list of same-document ref actions.\n- `close`: close the task-owned browser session.\n\nThe tool does not accept CSS, XPath, JavaScript, cookie, storage, or network-inspection commands. Re-observe after navigation or a stale-observation response. Browser actions execute directly under the configured allow/deny policy. Sensitive fields expose only their semantic identity and values are redacted.\n\n## browser_automation\n\nReusable browser automations are strict JSON. They use semantic `role` plus `name` or `nameIncludes` targets and an exact `allowedDomains` list. Input templates use `${input.name}`. Save a draft, validate it, then run/test that exact saved revision when the user requests execution or testing. Never repeat a write merely to verify a recording. Only successful runs with explicit successCriteria mark a revision verified; never store passwords, one-time codes, recovery codes, or payment details.\n";
|
|
@@ -12,16 +12,16 @@ Available actions:
|
|
|
12
12
|
- \`click\`, \`fill\`, \`select\`: act on a semantic ref with its observation revision.
|
|
13
13
|
- \`press\`, \`scroll\`: act on an optional ref or the active page.
|
|
14
14
|
- \`wait\`: wait for page idle, text, or a ref visibility state.
|
|
15
|
-
- \`upload\`: attach workspace files to a file input.
|
|
15
|
+
- \`upload\`: attach workspace files to a file input.
|
|
16
16
|
- \`tabs\`: list, create, activate, or close task-owned tabs.
|
|
17
17
|
- \`sequence\`: run a short list of same-document ref actions.
|
|
18
18
|
- \`close\`: close the task-owned browser session.
|
|
19
19
|
|
|
20
|
-
The tool does not accept CSS, XPath, JavaScript, cookie, storage, or network-inspection commands. Re-observe after navigation or a stale-observation response.
|
|
20
|
+
The tool does not accept CSS, XPath, JavaScript, cookie, storage, or network-inspection commands. Re-observe after navigation or a stale-observation response. Browser actions execute directly under the configured allow/deny policy. Sensitive fields expose only their semantic identity and values are redacted.
|
|
21
21
|
|
|
22
22
|
## browser_automation
|
|
23
23
|
|
|
24
|
-
Reusable browser automations are strict JSON. They use semantic \`role\` plus \`name\` or \`nameIncludes\` targets and an exact \`allowedDomains\` list. Input templates use \`\${input.name}\`. Save
|
|
24
|
+
Reusable browser automations are strict JSON. They use semantic \`role\` plus \`name\` or \`nameIncludes\` targets and an exact \`allowedDomains\` list. Input templates use \`\${input.name}\`. Save a draft, validate it, then run/test that exact saved revision when the user requests execution or testing. Never repeat a write merely to verify a recording. Only successful runs with explicit successCriteria mark a revision verified; never store passwords, one-time codes, recovery codes, or payment details.
|
|
25
25
|
`;
|
|
26
26
|
//#endregion
|
|
27
27
|
export { browserUseManual };
|
|
@@ -56,7 +56,7 @@ function presentResult(result) {
|
|
|
56
56
|
content: contentWithVisual([`[${result.error.code}] ${result.error.message}`, observation ? formatBrowserObservation(observation) : ""].filter(Boolean).join("\n\n"), observation),
|
|
57
57
|
details: {
|
|
58
58
|
ok: false,
|
|
59
|
-
kind:
|
|
59
|
+
kind: "browser_error",
|
|
60
60
|
error: stripVisual(result.error)
|
|
61
61
|
}
|
|
62
62
|
};
|
|
@@ -3,13 +3,11 @@ import { type Static } from '@sinclair/typebox';
|
|
|
3
3
|
export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TObject<{
|
|
4
4
|
action: import("@sinclair/typebox").TLiteral<"observe">;
|
|
5
5
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
6
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
7
6
|
} & {
|
|
8
7
|
visual: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"never">, import("@sinclair/typebox").TLiteral<"auto">, import("@sinclair/typebox").TLiteral<"always">]>>;
|
|
9
8
|
}>, import("@sinclair/typebox").TObject<{
|
|
10
9
|
action: import("@sinclair/typebox").TLiteral<"navigate">;
|
|
11
10
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
12
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
13
11
|
} & {
|
|
14
12
|
url: import("@sinclair/typebox").TString;
|
|
15
13
|
expect: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
|
|
@@ -22,7 +20,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
22
20
|
}>, import("@sinclair/typebox").TObject<{
|
|
23
21
|
action: import("@sinclair/typebox").TLiteral<"click">;
|
|
24
22
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
25
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
26
23
|
} & {
|
|
27
24
|
revision: import("@sinclair/typebox").TNumber;
|
|
28
25
|
ref: import("@sinclair/typebox").TString;
|
|
@@ -36,7 +33,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
36
33
|
}>, import("@sinclair/typebox").TObject<{
|
|
37
34
|
action: import("@sinclair/typebox").TLiteral<"fill">;
|
|
38
35
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
39
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
40
36
|
} & {
|
|
41
37
|
revision: import("@sinclair/typebox").TNumber;
|
|
42
38
|
ref: import("@sinclair/typebox").TString;
|
|
@@ -52,7 +48,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
52
48
|
}>, import("@sinclair/typebox").TObject<{
|
|
53
49
|
action: import("@sinclair/typebox").TLiteral<"select">;
|
|
54
50
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
55
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
56
51
|
} & {
|
|
57
52
|
revision: import("@sinclair/typebox").TNumber;
|
|
58
53
|
ref: import("@sinclair/typebox").TString;
|
|
@@ -67,7 +62,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
67
62
|
}>, import("@sinclair/typebox").TObject<{
|
|
68
63
|
action: import("@sinclair/typebox").TLiteral<"press">;
|
|
69
64
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
70
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
71
65
|
} & {
|
|
72
66
|
revision: import("@sinclair/typebox").TNumber;
|
|
73
67
|
ref: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
@@ -82,7 +76,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
82
76
|
}>, import("@sinclair/typebox").TObject<{
|
|
83
77
|
action: import("@sinclair/typebox").TLiteral<"scroll">;
|
|
84
78
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
85
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
86
79
|
} & {
|
|
87
80
|
revision: import("@sinclair/typebox").TNumber;
|
|
88
81
|
ref: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
@@ -97,7 +90,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
97
90
|
}>, import("@sinclair/typebox").TObject<{
|
|
98
91
|
action: import("@sinclair/typebox").TLiteral<"wait">;
|
|
99
92
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
100
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
101
93
|
} & {
|
|
102
94
|
revision: import("@sinclair/typebox").TNumber;
|
|
103
95
|
condition: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"page_idle">, import("@sinclair/typebox").TLiteral<"text">, import("@sinclair/typebox").TLiteral<"visible">, import("@sinclair/typebox").TLiteral<"hidden">]>;
|
|
@@ -107,7 +99,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
107
99
|
}>, import("@sinclair/typebox").TObject<{
|
|
108
100
|
action: import("@sinclair/typebox").TLiteral<"upload">;
|
|
109
101
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
110
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
111
102
|
} & {
|
|
112
103
|
revision: import("@sinclair/typebox").TNumber;
|
|
113
104
|
ref: import("@sinclair/typebox").TString;
|
|
@@ -122,7 +113,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
122
113
|
}>, import("@sinclair/typebox").TObject<{
|
|
123
114
|
action: import("@sinclair/typebox").TLiteral<"tabs">;
|
|
124
115
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
125
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
126
116
|
} & {
|
|
127
117
|
operation: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"list">, import("@sinclair/typebox").TLiteral<"create">, import("@sinclair/typebox").TLiteral<"activate">, import("@sinclair/typebox").TLiteral<"close">]>;
|
|
128
118
|
tabId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
@@ -130,7 +120,6 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
130
120
|
}>, import("@sinclair/typebox").TObject<{
|
|
131
121
|
action: import("@sinclair/typebox").TLiteral<"sequence">;
|
|
132
122
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
133
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
134
123
|
} & {
|
|
135
124
|
revision: import("@sinclair/typebox").TNumber;
|
|
136
125
|
steps: import("@sinclair/typebox").TArray<import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TObject<{
|
|
@@ -199,12 +188,10 @@ export declare const BrowserUseActionSchema: import("@sinclair/typebox").TUnion<
|
|
|
199
188
|
}>, import("@sinclair/typebox").TObject<{
|
|
200
189
|
action: import("@sinclair/typebox").TLiteral<"close">;
|
|
201
190
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
202
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
203
191
|
}>]>;
|
|
204
192
|
export declare const BrowserUseSchema: import("@sinclair/typebox").TObject<{
|
|
205
193
|
action: import("@sinclair/typebox").TUnion<[import("@sinclair/typebox").TLiteral<"observe">, import("@sinclair/typebox").TLiteral<"navigate">, import("@sinclair/typebox").TLiteral<"click">, import("@sinclair/typebox").TLiteral<"fill">, import("@sinclair/typebox").TLiteral<"select">, import("@sinclair/typebox").TLiteral<"press">, import("@sinclair/typebox").TLiteral<"scroll">, import("@sinclair/typebox").TLiteral<"wait">, import("@sinclair/typebox").TLiteral<"upload">, import("@sinclair/typebox").TLiteral<"tabs">, import("@sinclair/typebox").TLiteral<"sequence">, import("@sinclair/typebox").TLiteral<"close">]>;
|
|
206
194
|
sessionId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
207
|
-
approvalId: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
208
195
|
revision: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
209
196
|
ref: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
210
197
|
expect: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TObject<{
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Type } from "@sinclair/typebox";
|
|
2
2
|
import { Value } from "@sinclair/typebox/value";
|
|
3
3
|
//#region src/agent/tools/browser/tool/schemas.ts
|
|
4
|
-
const ApprovalId = Type.Optional(Type.String({ description: "One-time local-owner approval id for this exact action." }));
|
|
5
4
|
const SessionId = Type.Optional(Type.String({ description: "Browser session id returned by the previous observation." }));
|
|
6
5
|
const Revision = Type.Number({ description: "Observation revision used to resolve element refs." });
|
|
7
6
|
const Ref = Type.String({
|
|
@@ -20,7 +19,6 @@ function action(name, properties) {
|
|
|
20
19
|
return Type.Object({
|
|
21
20
|
action: Type.Literal(name),
|
|
22
21
|
sessionId: SessionId,
|
|
23
|
-
approvalId: ApprovalId,
|
|
24
22
|
...properties
|
|
25
23
|
}, { additionalProperties: false });
|
|
26
24
|
}
|
|
@@ -168,7 +166,6 @@ const BrowserUseSchema = Type.Object({
|
|
|
168
166
|
Type.Literal("close")
|
|
169
167
|
]),
|
|
170
168
|
sessionId: SessionId,
|
|
171
|
-
approvalId: ApprovalId,
|
|
172
169
|
revision: Type.Optional(Revision),
|
|
173
170
|
ref: Type.Optional(Ref),
|
|
174
171
|
expect: Type.Optional(ExpectationSchema),
|
|
@@ -295,7 +292,6 @@ function decodeBrowserUseInput(value) {
|
|
|
295
292
|
if (!isRecord(value) || !isBrowserActionName(value.action)) return null;
|
|
296
293
|
const candidate = { action: value.action };
|
|
297
294
|
copyNonEmptyString(value, candidate, "sessionId");
|
|
298
|
-
copyNonEmptyString(value, candidate, "approvalId");
|
|
299
295
|
for (const field of ACTION_FIELDS[value.action]) {
|
|
300
296
|
const fieldValue = value[field];
|
|
301
297
|
if (fieldValue === void 0 || fieldValue === null) continue;
|