@xopcai/xopc 0.0.368 → 0.0.369
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-ext/_locales/en/messages.json +30 -1
- package/dist/browser-ext/_locales/zh_CN/messages.json +30 -1
- package/dist/browser-ext/dist/assets/{auth-beGtbLPS.js → page-context-BeworyNB.js} +1331 -1355
- package/dist/browser-ext/dist/assets/{sidepanel-BT0ojEzG.css → sidepanel-Ccl0eSa1.css} +43 -0
- package/dist/browser-ext/dist/background.js +680 -182
- package/dist/browser-ext/dist/sidepanel.html +2 -2
- package/dist/browser-ext/dist/sidepanel.js +372 -193
- package/dist/browser-ext/manifest.json +2 -8
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/{activity-store-D9ZMh5ZR.js → activity-store-BQ3PMp2q.js} +1 -1
- package/dist/gateway/static/root/assets/{agent-avatar-dicebear-Bj3rvcTm.js → agent-avatar-dicebear-Stj17S9u.js} +1 -1
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-CCRfvuov.js +1 -0
- package/dist/gateway/static/root/assets/{agent-plugin-dialog-BjdgKCnk.js → agent-plugin-dialog-CRlzd8El.js} +1 -1
- package/dist/gateway/static/root/assets/agents-_ug2LCQb.js +66 -0
- package/dist/gateway/static/root/assets/{agents-admin-api-B5QDepbW.js → agents-admin-api-HBZvVyzF.js} +1 -1
- package/dist/gateway/static/root/assets/{app-management-settings-panel-2rTEtB_X.js → app-management-settings-panel-pJFe6M6c.js} +1 -1
- package/dist/gateway/static/root/assets/{appearance-settings-BPVBS80k.js → appearance-settings-BrxSpBWa.js} +1 -1
- package/dist/gateway/static/root/assets/{apps-page-1FuKlZYP.js → apps-page-DegSl5Lh.js} +1 -1
- package/dist/gateway/static/root/assets/{archive-plugin-BbT1O7oK.js → archive-plugin-BO3I9zRP.js} +1 -1
- package/dist/gateway/static/root/assets/{automations-page-DjwP_h8h.js → automations-page-BZQ_VHjp.js} +1 -1
- package/dist/gateway/static/root/assets/automations-workspace-BQywN6nG.js +6 -0
- package/dist/gateway/static/root/assets/{binary-plugins-gS7Wg68v.js → binary-plugins-gFD7vH9R.js} +1 -1
- package/dist/gateway/static/root/assets/{block-editor-DbwMaivP.js → block-editor-3XjFv3kK.js} +3 -3
- package/dist/gateway/static/root/assets/browser-automation-inputs-DxRoKFun.js +1 -0
- package/dist/gateway/static/root/assets/browser-automations-page-yiEuG1v7.js +3 -0
- package/dist/gateway/static/root/assets/browser-settings-page-ch2_bQF0.js +3 -0
- package/dist/gateway/static/root/assets/{capabilities-Dd1PZRQw.js → capabilities-B48r0DDa.js} +1 -1
- package/dist/gateway/static/root/assets/{capabilities-Lswa0hv8.js → capabilities-DO-vfNKI.js} +1 -1
- package/dist/gateway/static/root/assets/{capabilities-page-djBsC_tG.js → capabilities-page-BEnFt_dj.js} +2 -2
- package/dist/gateway/static/root/assets/{capabilities-settings-panel-DTHR1DrJ.js → capabilities-settings-panel-NckMsJ5N.js} +4 -4
- package/dist/gateway/static/root/assets/{capability-header-actions-494dnJsj.js → capability-header-actions-CY8KCO9j.js} +1 -1
- package/dist/gateway/static/root/assets/channels-CHOS2CGa.js +1 -0
- package/dist/gateway/static/root/assets/{chat-terminal-dock-Bl7-Zk-x.js → chat-terminal-dock-Bwknnb4c.js} +2 -2
- package/dist/gateway/static/root/assets/{computer-settings-page-ByExKr_6.js → computer-settings-page-B2xeFCSt.js} +1 -1
- package/dist/gateway/static/root/assets/{connector-service-page-BK_5aFxO.js → connector-service-page-C83DnAqK.js} +1 -1
- package/dist/gateway/static/root/assets/connectors-page-D6wTmWA6.js +2 -0
- package/dist/gateway/static/root/assets/{date-picker-BXJv6472.js → date-picker-Bq9GR-DU.js} +1 -1
- package/dist/gateway/static/root/assets/{dependency-picker-CWbbhKZa.js → dependency-picker-BTERkA9X.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-DWNEC3mk.js → desktop-pet-BwsbQw8V.js} +1 -1
- package/dist/gateway/static/root/assets/{desktop-pet-settings-BYwlxGiN.js → desktop-pet-settings-sL9i44pP.js} +1 -1
- package/dist/gateway/static/root/assets/device-pairing-wizard-DI2bqaKO.js +1 -0
- package/dist/gateway/static/root/assets/{directory-picker-path-field-D26tSY8Z.js → directory-picker-path-field-BFlDmsS5.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-debug-page-xYIGNegB.js → extension-debug-page-7Z1Tp7-w.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-page-BoQ1-3AW.js → extension-page-uGqvb8QX.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-provider-Cw_JU6RR.js → extension-provider-xlSyz4Z6.js} +1 -1
- package/dist/gateway/static/root/assets/{extension-settings-page-BH5TFGPe.js → extension-settings-page-CKHgrjOs.js} +1 -1
- package/dist/gateway/static/root/assets/{fallback-models-04Wqmw14.js → fallback-models-BwQg0G1D.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-config-swr-CNf0ZeRw.js → gateway-config-swr-7RKqgU_U.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-realtime-C4KtwMY3.js → gateway-realtime-DDWymi-o.js} +1 -1
- package/dist/gateway/static/root/assets/gateway-realtime-store-BheKaO6d.js +1 -0
- package/dist/gateway/static/root/assets/{gateway-settings-BqvCJVUC.js → gateway-settings-BAQnSEMJ.js} +1 -1
- package/dist/gateway/static/root/assets/{gateway-startup-retry-DIxHacUw.js → gateway-startup-retry-jiV5sbEX.js} +1 -1
- package/dist/gateway/static/root/assets/home-page-BZDjvwet.js +2 -0
- package/dist/gateway/static/root/assets/{image-generation-api-CS0q0Kto.js → image-generation-api-DO4mXb6I.js} +1 -1
- package/dist/gateway/static/root/assets/{imports-page-n_rSLo6b.js → imports-page-re1_Nh64.js} +1 -1
- package/dist/gateway/static/root/assets/index-BB1z8QxA.js +106 -0
- package/dist/gateway/static/root/assets/index-D9R6tU7r.css +1 -0
- package/dist/gateway/static/root/assets/{keyboard-shortcuts-settings-tVkrHbYF.js → keyboard-shortcuts-settings-B_xja9LN.js} +1 -1
- package/dist/gateway/static/root/assets/{local-app-workbench-page-L8tYuas0.js → local-app-workbench-page-CErdCrPX.js} +1 -1
- package/dist/gateway/static/root/assets/{local-apps-page-SSJ9B1bH.js → local-apps-page-BPL2RXdK.js} +1 -1
- package/dist/gateway/static/root/assets/{local-session-drafts-B_McurIj.js → local-session-drafts-DRPsDCj1.js} +1 -1
- package/dist/gateway/static/root/assets/{locale-store-D6_rF-qT.js → locale-store-COVscAqC.js} +1 -1
- package/dist/gateway/static/root/assets/{logs-page-BWD63eg1.js → logs-page-DL-BM0xq.js} +2 -2
- package/dist/gateway/static/root/assets/management-settings-BOVRHhXk.js +1 -0
- package/dist/gateway/static/root/assets/{markdown-view-_xwUuKce.js → markdown-view-B_qnAcW2.js} +1 -1
- package/dist/gateway/static/root/assets/{media-plugins-C631CDu4.js → media-plugins-C0Pc-wSY.js} +1 -1
- package/dist/gateway/static/root/assets/messages-Bb2GK-Nj.js +3 -0
- package/dist/gateway/static/root/assets/models-hub-cache-DyPPksMP.js +1 -0
- package/dist/gateway/static/root/assets/note-detail-panel-DJJK2yL4.js +5 -0
- package/dist/gateway/static/root/assets/{notes-api-vIZWQ9Yt.js → notes-api-sBgBcyLW.js} +1 -1
- package/dist/gateway/static/root/assets/notes-page-oCxO7ZTv.js +8 -0
- package/dist/gateway/static/root/assets/{page-context-capture-button-BlPDM7GM.js → page-context-capture-button-tf77xIz3.js} +1 -1
- package/dist/gateway/static/root/assets/personal-page-DOk9gUK5.js +1 -0
- package/dist/gateway/static/root/assets/{product-open-page-BJjWvoER.js → product-open-page-Dc3Y-Y7J.js} +1 -1
- package/dist/gateway/static/root/assets/{product-preview-content-CEd4E7Gf.js → product-preview-content-MG_hmVA3.js} +1 -1
- package/dist/gateway/static/root/assets/{project-detail-page-DM4_VLNX.js → project-detail-page-3uAzbW4T.js} +1 -1
- package/dist/gateway/static/root/assets/{projects-page-lszV8HuS.js → projects-page-MllfNLGA.js} +1 -1
- package/dist/gateway/static/root/assets/{registry-api-uJnjqTcn.js → registry-api-Bh9UBb-f.js} +1 -1
- package/dist/gateway/static/root/assets/{remote-access-hub-O6j9CWX0.js → remote-access-hub-Bs-C70el.js} +1 -1
- package/dist/gateway/static/root/assets/{runtime-tools-settings-panel-CjnNQYW8.js → runtime-tools-settings-panel-DpJ5jNzR.js} +1 -1
- package/dist/gateway/static/root/assets/{scenes-page-B3T3dDOR.js → scenes-page-Dpxamkw6.js} +2 -2
- package/dist/gateway/static/root/assets/{schema-form-DT3c4Vd_.js → schema-form-DUeyQ1zh.js} +1 -1
- package/dist/gateway/static/root/assets/{send-session-input-CICkGwn-.js → send-session-input-Dgn4BRk_.js} +1 -1
- package/dist/gateway/static/root/assets/{session-api-BcQodoAu.js → session-api-yjpEz4pP.js} +1 -1
- package/dist/gateway/static/root/assets/{session-manager-B0tOE4qz.js → session-manager-LInoZ_Yi.js} +1 -1
- package/dist/gateway/static/root/assets/sessions-page-BCVlNHtq.js +1 -0
- package/dist/gateway/static/root/assets/settings-advanced-gate-Bz-ue9fZ.js +1 -0
- package/dist/gateway/static/root/assets/{settings-page-D-PZjcss.js → settings-page-DouHuBXt.js} +1 -1
- package/dist/gateway/static/root/assets/{setup-status-panel-Ce3gTbT3.js → setup-status-panel-B5BoZ-1y.js} +1 -1
- package/dist/gateway/static/root/assets/{share-link-dialog-CWS8EkJS.js → share-link-dialog-DJTabLAM.js} +1 -1
- package/dist/gateway/static/root/assets/{share-preview-page-23NJAKiq.js → share-preview-page-jw7h4t_B.js} +1 -1
- package/dist/gateway/static/root/assets/{shares-api-iKhHbvBD.js → shares-api-c0dfWz6h.js} +1 -1
- package/dist/gateway/static/root/assets/shares-settings-BlnlzGjH.js +1 -0
- package/dist/gateway/static/root/assets/{skill-api-C-MWdQwl.js → skill-api-CfyX_0zN.js} +1 -1
- package/dist/gateway/static/root/assets/{skill-reload-api-iy02aB13.js → skill-reload-api-Ge5XymHa.js} +1 -1
- package/dist/gateway/static/root/assets/skills-page-DjdTeYzm.js +2 -0
- package/dist/gateway/static/root/assets/system-settings-panel-DWf256TE.js +1 -0
- package/dist/gateway/static/root/assets/{task-detail-page-B2SIEyhZ.js → task-detail-page-DE6N1ZQe.js} +2 -2
- package/dist/gateway/static/root/assets/{task-expandable-content-D2LRR4BC.js → task-expandable-content-hYcgCEE3.js} +1 -1
- package/dist/gateway/static/root/assets/{theme-store-BJfhz-Zq.js → theme-store-BOfCKgl6.js} +1 -1
- package/dist/gateway/static/root/assets/{tracing-settings-page-aHD56iaS.js → tracing-settings-page-DDWFXcST.js} +1 -1
- package/dist/gateway/static/root/assets/{tunnel-api-BCf8CfTC.js → tunnel-api-BBm9esrY.js} +1 -1
- package/dist/gateway/static/root/assets/{usage-page-B4XUkuJd.js → usage-page-C93x89uW.js} +1 -1
- package/dist/gateway/static/root/assets/{use-autosave-BnnTw9RE.js → use-autosave-CSckcjFx.js} +1 -1
- package/dist/gateway/static/root/assets/{use-composer-attachments-Itczg0y1.js → use-composer-attachments-C7Gaaai1.js} +1 -1
- package/dist/gateway/static/root/assets/{use-file-preview-expanded-DJ5vyrvM.js → use-file-preview-expanded-Db2VYg1b.js} +2 -2
- package/dist/gateway/static/root/assets/user-model-page-B74LkVbp.js +2 -0
- package/dist/gateway/static/root/assets/{voice-api-key-field-BphJku8x.js → voice-api-key-field-GEw5x2fU.js} +1 -1
- package/dist/gateway/static/root/assets/{voice-config-api-B4u4coFj.js → voice-config-api-3yyVwXln.js} +1 -1
- package/dist/gateway/static/root/assets/{work-discovery-overlay-jgOEP1xw.js → work-discovery-overlay-Bs2MKiTv.js} +1 -1
- package/dist/gateway/static/root/assets/{workflow-run-setup-panel-N69crzRu.js → workflow-run-setup-panel-BR_U_snV.js} +2 -2
- package/dist/gateway/static/root/assets/workflows-page-Crk4SxtC.js +6 -0
- package/dist/gateway/static/root/assets/{workspace-api-Lzv6GDO8.js → workspace-api-Si2o3inw.js} +2 -2
- package/dist/gateway/static/root/index.html +27 -27
- package/dist/package.js +1 -1
- package/dist/packages/browser-control-contract/src/automation.js +213 -0
- package/dist/packages/browser-control-contract/src/index.js +43 -12
- package/dist/packages/gateway-contract/src/gateway-scopes.js +3 -1
- package/dist/packages/gateway-contract/src/notifications.js +1 -0
- package/dist/packages/realtime-protocol/src/voice.js +9 -3
- package/dist/src/agent/coding/repository-instructions.js +8 -1
- package/dist/src/agent/tool-manuals/browser.d.ts +1 -1
- package/dist/src/agent/tool-manuals/browser.js +3 -3
- package/dist/src/agent/tools/browser/tool/browser-use-tool.js +1 -1
- package/dist/src/agent/tools/browser/tool/schemas.d.ts +0 -13
- package/dist/src/agent/tools/browser/tool/schemas.js +0 -4
- package/dist/src/agent/tools/browser-automation-tool.d.ts +74 -1
- package/dist/src/agent/tools/browser-automation-tool.js +52 -6
- package/dist/src/automations/capabilities/write.d.ts +2 -0
- package/dist/src/automations/domain/types.d.ts +11 -0
- package/dist/src/automations/domain/validation.d.ts +4 -0
- package/dist/src/automations/domain/validation.js +1 -0
- package/dist/src/automations/draft/automation-draft-validator.d.ts +1 -0
- package/dist/src/automations/service/action-executor.js +11 -3
- package/dist/src/automations/service/automation-service.d.ts +1 -0
- package/dist/src/automations/service/automation-service.js +13 -1
- package/dist/src/bootstrap/application-state.js +1 -1
- package/dist/src/browser/automations/repository.d.ts +4 -0
- package/dist/src/browser/automations/repository.js +27 -6
- package/dist/src/browser/automations/runner.js +101 -9
- package/dist/src/browser/automations/runtime-service.js +4 -3
- package/dist/src/browser/automations/schema.d.ts +84 -1
- package/dist/src/browser/automations/schema.js +68 -2
- package/dist/src/browser/automations/service.d.ts +78 -1
- package/dist/src/browser/automations/service.js +36 -4
- package/dist/src/browser/automations/types.d.ts +19 -4
- package/dist/src/browser/drivers/extension-driver.d.ts +5 -5
- package/dist/src/browser/drivers/extension-driver.js +10 -10
- package/dist/src/browser/drivers/playwright-driver.js +46 -0
- package/dist/src/browser/index.d.ts +1 -2
- package/dist/src/browser/index.js +2 -3
- package/dist/src/browser/policy/browser-policy.d.ts +0 -1
- package/dist/src/browser/policy/browser-policy.js +1 -4
- package/dist/src/browser/providers/index.d.ts +1 -1
- package/dist/src/browser/providers/index.js +2 -2
- package/dist/src/browser/providers/realtime-extension.d.ts +1 -1
- package/dist/src/browser/providers/realtime-extension.js +5 -1
- package/dist/src/browser/recordings/service.d.ts +22 -0
- package/dist/src/browser/recordings/service.js +160 -0
- package/dist/src/browser/runtime/browser-runtime.d.ts +1 -1
- package/dist/src/browser/runtime/browser-runtime.js +19 -56
- package/dist/src/cli/commands/doctor/checks/database-schema.js +1 -1
- package/dist/src/computer/runtime.js +2 -2
- package/dist/src/config/schema.d.ts +12 -12
- package/dist/src/config/schema.js +12 -12
- package/dist/src/endpoint-tools/policy.js +4 -3
- package/dist/src/gateway/hono/routes/browser-automations.js +23 -7
- package/dist/src/gateway/hono/routes/browser-recordings.d.ts +3 -0
- package/dist/src/gateway/hono/routes/browser-recordings.js +78 -0
- package/dist/src/gateway/hono/routes/browser.js +1 -34
- package/dist/src/gateway/hono/routes/device-push.js +1 -0
- package/dist/src/gateway/hono/routes/home.js +1 -1
- package/dist/src/gateway/hono/routes/lazy-bundles.js +8 -0
- package/dist/src/gateway/hono/routes/personal-agent.js +35 -0
- package/dist/src/gateway/hono/routes/voice.js +5 -2
- package/dist/src/gateway/security/gateway-scopes.js +3 -1
- package/dist/src/gateway/service/agent-runner.js +1 -0
- package/dist/src/home-intelligence/generator.d.ts +2 -0
- package/dist/src/home-intelligence/generator.js +53 -4
- package/dist/src/home-intelligence/host.d.ts +2 -1
- package/dist/src/home-intelligence/host.js +23 -3
- package/dist/src/migrations/cloud-public-models.js +1 -0
- package/dist/src/notifications/clarification.d.ts +5 -0
- package/dist/src/notifications/clarification.js +66 -0
- package/dist/src/notifications/device-store.js +1 -0
- package/dist/src/notifications/harmony-push.d.ts +1 -0
- package/dist/src/notifications/harmony-push.js +3 -1
- package/dist/src/notifications/planner.d.ts +1 -0
- package/dist/src/notifications/service.d.ts +2 -2
- package/dist/src/notifications/service.js +20 -4
- package/dist/src/notifications/store.d.ts +2 -0
- package/dist/src/notifications/store.js +7 -1
- package/dist/src/notifications/types.d.ts +1 -0
- package/dist/src/notifications/types.js +1 -0
- package/dist/src/storage/sqlite/clarification-wait-repository.js +1 -0
- package/dist/src/storage/sqlite/migrations/238_browser_record_replay.sql +45 -0
- package/dist/src/storage/sqlite/migrations/239_clarification_notifications.sql +26 -0
- package/dist/src/storage/sqlite/migrations/runner.d.ts +1 -1
- package/dist/src/storage/sqlite/migrations/runner.js +3 -3
- package/dist/src/storage/sqlite/schema.js +1 -1
- package/dist/src/tasks/home-query-service.d.ts +1 -1
- package/dist/src/tasks/home-query-service.js +6 -2
- package/dist/src/voice/dashscope/streaming-tts-stream.d.ts +2 -1
- package/dist/src/voice/dashscope/streaming-tts-stream.js +135 -79
- package/dist/src/voice/realtime/agentEngine.js +78 -1
- package/dist/src/voice/realtime/speakable-segmenter.d.ts +2 -0
- package/dist/src/voice/realtime/speakable-segmenter.js +30 -0
- package/dist/src/voice/tts/providers/alibaba-speech.js +17 -1
- package/dist/src/voice/tts/speak-core.d.ts +3 -0
- package/dist/src/voice/tts/speak-core.js +21 -1
- package/dist/src/voice/tts/speech-provider-types.d.ts +8 -0
- package/package.json +3 -3
- package/dist/gateway/static/root/assets/agent-defaults-settings-panel-D0tD3kCe.js +0 -1
- package/dist/gateway/static/root/assets/agents-CWXKzX5L.js +0 -66
- package/dist/gateway/static/root/assets/automations-workspace-C9ZT6TAU.js +0 -6
- package/dist/gateway/static/root/assets/browser-automation-inputs-CXG_JveC.js +0 -1
- package/dist/gateway/static/root/assets/browser-automations-page-CJc7OKR3.js +0 -3
- package/dist/gateway/static/root/assets/browser-settings-page-BjaTEUR8.js +0 -3
- package/dist/gateway/static/root/assets/channels-BaYRJtz3.js +0 -1
- package/dist/gateway/static/root/assets/connectors-page-DyqRwSH2.js +0 -2
- package/dist/gateway/static/root/assets/device-pairing-wizard-BAc4F35b.js +0 -1
- package/dist/gateway/static/root/assets/gateway-realtime-store-ChZUzVbe.js +0 -1
- package/dist/gateway/static/root/assets/home-page-DhPPLbdD.js +0 -2
- package/dist/gateway/static/root/assets/index-CK4O2xMa.css +0 -1
- package/dist/gateway/static/root/assets/index-DpLD-7Hc.js +0 -106
- package/dist/gateway/static/root/assets/management-settings-aKSibklM.js +0 -1
- package/dist/gateway/static/root/assets/messages-Bnx3GMDB.js +0 -3
- package/dist/gateway/static/root/assets/models-hub-cache-Di83uIPw.js +0 -1
- package/dist/gateway/static/root/assets/note-detail-panel-B_fve3_2.js +0 -5
- package/dist/gateway/static/root/assets/notes-page-DnChyp8S.js +0 -8
- package/dist/gateway/static/root/assets/personal-page-BM2qAgg_.js +0 -1
- package/dist/gateway/static/root/assets/sessions-page-BIixHXlQ.js +0 -1
- package/dist/gateway/static/root/assets/settings-advanced-gate-49SUEWPh.js +0 -1
- package/dist/gateway/static/root/assets/shares-settings-CNEZjF66.js +0 -1
- package/dist/gateway/static/root/assets/skills-page-BndNdYsJ.js +0 -2
- package/dist/gateway/static/root/assets/system-settings-panel-EpZeMlvp.js +0 -1
- package/dist/gateway/static/root/assets/user-model-page-Bz5lkFG9.js +0 -2
- package/dist/gateway/static/root/assets/workflows-page-CPCgOfHh.js +0 -6
- package/dist/src/browser/policy/approval-store.d.ts +0 -17
- package/dist/src/browser/policy/approval-store.js +0 -62
- package/dist/src/browser/providers/extension-ws-acquire.d.ts +0 -19
- package/dist/src/browser/providers/extension-ws-acquire.js +0 -78
- package/dist/src/browser/providers/extension.d.ts +0 -61
- package/dist/src/browser/providers/extension.js +0 -378
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BrowserAutomationDefinitionSchema } from "./schema.js";
|
|
2
2
|
import { resolveBrowserAutomationInputs } from "./runner.js";
|
|
3
|
-
import { appendBrowserAutomationRunEvent, deleteBrowserAutomation, getBrowserAutomation, getBrowserAutomationRun, hasActiveBrowserAutomationRuns, listActiveBrowserAutomationRuns, listBrowserAutomationRunEvents, listBrowserAutomationRuns, listBrowserAutomations, nextBrowserAutomationRunEventSeq, saveBrowserAutomation, saveBrowserAutomationRun } from "./repository.js";
|
|
3
|
+
import { appendBrowserAutomationRunEvent, deleteBrowserAutomation, findBrowserAutomationRunByRequest, getBrowserAutomation, getBrowserAutomationRun, getBrowserAutomationVersion, hasActiveBrowserAutomationRuns, listActiveBrowserAutomationRuns, listBrowserAutomationRunEvents, listBrowserAutomationRuns, listBrowserAutomationVersions, listBrowserAutomations, nextBrowserAutomationRunEventSeq, saveBrowserAutomation, saveBrowserAutomationRun, verifyBrowserAutomationRun } from "./repository.js";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
//#region src/browser/automations/service.ts
|
|
6
6
|
var BrowserAutomationService = class {
|
|
@@ -15,7 +15,8 @@ var BrowserAutomationService = class {
|
|
|
15
15
|
saveBrowserAutomationRun({
|
|
16
16
|
...run,
|
|
17
17
|
status: "failed",
|
|
18
|
-
|
|
18
|
+
businessOutcome: "unknown",
|
|
19
|
+
error: "Gateway stopped before the run completed. Check the business result before running again.",
|
|
19
20
|
endedAtMs,
|
|
20
21
|
durationMs: run.startedAtMs ? endedAtMs - run.startedAtMs : void 0
|
|
21
22
|
});
|
|
@@ -27,6 +28,9 @@ var BrowserAutomationService = class {
|
|
|
27
28
|
get(id) {
|
|
28
29
|
return getBrowserAutomation(id);
|
|
29
30
|
}
|
|
31
|
+
versions(id) {
|
|
32
|
+
return listBrowserAutomationVersions(id);
|
|
33
|
+
}
|
|
30
34
|
listRuns(automationId) {
|
|
31
35
|
return listBrowserAutomationRuns(automationId);
|
|
32
36
|
}
|
|
@@ -43,10 +47,13 @@ var BrowserAutomationService = class {
|
|
|
43
47
|
const parsed = BrowserAutomationDefinitionSchema.parse(input.definition);
|
|
44
48
|
if (input.expectedId && parsed.id !== input.expectedId) throw new Error("Browser automation id cannot change.");
|
|
45
49
|
const existing = getBrowserAutomation(parsed.id);
|
|
50
|
+
if (input.expectedRevision !== void 0 && existing?.revision !== input.expectedRevision) throw new Error("Browser automation revision changed. Reload before saving.");
|
|
51
|
+
const changed = !existing || canonical(existing.definition) !== canonical(parsed);
|
|
46
52
|
const now = Date.now();
|
|
47
53
|
const automation = {
|
|
48
54
|
id: parsed.id,
|
|
49
|
-
|
|
55
|
+
verified: !changed && !!existing?.verified,
|
|
56
|
+
revision: (existing?.revision ?? 0) + (changed ? 1 : 0),
|
|
50
57
|
status: input.status ?? existing?.status ?? "enabled",
|
|
51
58
|
definition: parsed,
|
|
52
59
|
createdAtMs: existing?.createdAtMs ?? now,
|
|
@@ -60,12 +67,22 @@ var BrowserAutomationService = class {
|
|
|
60
67
|
return deleteBrowserAutomation(id);
|
|
61
68
|
}
|
|
62
69
|
startRun(automationId, inputs, context) {
|
|
63
|
-
const
|
|
70
|
+
const requestId = context?.clientRequestId;
|
|
71
|
+
if (requestId) {
|
|
72
|
+
const previous = findBrowserAutomationRunByRequest(requestId);
|
|
73
|
+
if (previous) {
|
|
74
|
+
const contextualInputs = mergeBrowserAutomationTriggerInputs(previous.definition, inputs, context);
|
|
75
|
+
if (previous.automationId !== automationId || context?.revision !== void 0 && context.revision !== previous.automationRevision || canonical(previous.inputs) !== canonical(resolveBrowserAutomationInputs(previous.definition, contextualInputs))) throw new Error("Run request id was reused with a different automation, revision or inputs.");
|
|
76
|
+
return previous;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const automation = context?.revision !== void 0 ? getBrowserAutomationVersion(automationId, context.revision) : getBrowserAutomation(automationId);
|
|
64
80
|
if (!automation) throw new Error("Browser automation not found.");
|
|
65
81
|
if (automation.status !== "enabled") throw new Error("Browser automation is disabled.");
|
|
66
82
|
const contextualInputs = mergeBrowserAutomationTriggerInputs(automation.definition, inputs, context);
|
|
67
83
|
const resolvedInputs = resolveBrowserAutomationInputs(automation.definition, contextualInputs);
|
|
68
84
|
const run = {
|
|
85
|
+
clientRequestId: requestId,
|
|
69
86
|
id: randomUUID(),
|
|
70
87
|
automationId,
|
|
71
88
|
automationRevision: automation.revision,
|
|
@@ -93,6 +110,8 @@ var BrowserAutomationService = class {
|
|
|
93
110
|
}
|
|
94
111
|
async runAndWait(automationId, inputs, signal, context) {
|
|
95
112
|
const run = this.startRun(automationId, inputs, context);
|
|
113
|
+
const terminal = this.getRun(run.id);
|
|
114
|
+
if (terminal && !["queued", "running"].includes(terminal.status)) return terminal;
|
|
96
115
|
return new Promise((resolve) => {
|
|
97
116
|
const complete = (value) => {
|
|
98
117
|
signal?.removeEventListener("abort", onAbort);
|
|
@@ -131,6 +150,7 @@ var BrowserAutomationService = class {
|
|
|
131
150
|
this.addEvent(run.id, "run.started");
|
|
132
151
|
try {
|
|
133
152
|
const outcome = await this.executor({
|
|
153
|
+
runId: run.id,
|
|
134
154
|
definition: run.definition,
|
|
135
155
|
inputs: run.inputs,
|
|
136
156
|
signal: controller.signal,
|
|
@@ -142,6 +162,13 @@ var BrowserAutomationService = class {
|
|
|
142
162
|
status: controller.signal.aborted ? "cancelled" : outcome.ok ? "succeeded" : "failed",
|
|
143
163
|
result: outcome.result,
|
|
144
164
|
error: outcome.error,
|
|
165
|
+
businessOutcome: outcome.businessOutcome ?? (!outcome.ok && run.definition.steps.some((step) => [
|
|
166
|
+
"click",
|
|
167
|
+
"press",
|
|
168
|
+
"fill",
|
|
169
|
+
"select",
|
|
170
|
+
"check"
|
|
171
|
+
].includes(step.action)) ? "unknown" : "completed"),
|
|
145
172
|
endedAtMs,
|
|
146
173
|
durationMs: endedAtMs - startedAtMs
|
|
147
174
|
};
|
|
@@ -150,12 +177,14 @@ var BrowserAutomationService = class {
|
|
|
150
177
|
current = {
|
|
151
178
|
...current,
|
|
152
179
|
status: controller.signal.aborted ? "cancelled" : "failed",
|
|
180
|
+
businessOutcome: "unknown",
|
|
153
181
|
error: error instanceof Error ? error.message : String(error),
|
|
154
182
|
endedAtMs,
|
|
155
183
|
durationMs: endedAtMs - startedAtMs
|
|
156
184
|
};
|
|
157
185
|
}
|
|
158
186
|
saveBrowserAutomationRun(current);
|
|
187
|
+
verifyBrowserAutomationRun(current);
|
|
159
188
|
this.addEvent(run.id, "run.completed", {
|
|
160
189
|
status: current.status,
|
|
161
190
|
error: current.error
|
|
@@ -178,5 +207,8 @@ function mergeBrowserAutomationTriggerInputs(definition, inputs, context) {
|
|
|
178
207
|
for (const name of Object.keys(definition.inputs)) if (merged[name] === void 0 && candidates[name] !== void 0) merged[name] = candidates[name];
|
|
179
208
|
return merged;
|
|
180
209
|
}
|
|
210
|
+
function canonical(value) {
|
|
211
|
+
return JSON.stringify(value, (_key, item) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item);
|
|
212
|
+
}
|
|
181
213
|
//#endregion
|
|
182
214
|
export { BrowserAutomationService, mergeBrowserAutomationTriggerInputs };
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { BrowserExpectation, BrowserRiskLevel } from '@xopcai/browser-control-contract';
|
|
2
|
+
import type { BrowserSemanticTarget } from '@xopcai/browser-control-contract';
|
|
3
|
+
export type { BrowserSemanticTarget } from '@xopcai/browser-control-contract';
|
|
2
4
|
export type BrowserAutomationStatus = 'enabled' | 'disabled';
|
|
3
5
|
export type BrowserAutomationRunStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled';
|
|
4
6
|
export interface BrowserAutomationInputDefinition {
|
|
@@ -8,10 +10,11 @@ export interface BrowserAutomationInputDefinition {
|
|
|
8
10
|
description?: string;
|
|
9
11
|
choices?: Array<string | number | boolean>;
|
|
10
12
|
}
|
|
11
|
-
export interface
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
export interface BrowserAutomationAssertion {
|
|
14
|
+
target?: BrowserSemanticTarget;
|
|
15
|
+
field: 'text' | 'value' | 'url' | 'title' | 'checked';
|
|
16
|
+
equals?: string | boolean;
|
|
17
|
+
includes?: string;
|
|
15
18
|
}
|
|
16
19
|
export type BrowserAutomationStep = {
|
|
17
20
|
action: 'navigate';
|
|
@@ -32,6 +35,10 @@ export type BrowserAutomationStep = {
|
|
|
32
35
|
target: BrowserSemanticTarget;
|
|
33
36
|
value: string;
|
|
34
37
|
expect?: BrowserExpectation;
|
|
38
|
+
} | {
|
|
39
|
+
action: 'check';
|
|
40
|
+
target: BrowserSemanticTarget;
|
|
41
|
+
checked: boolean;
|
|
35
42
|
} | {
|
|
36
43
|
action: 'press';
|
|
37
44
|
target?: BrowserSemanticTarget;
|
|
@@ -57,10 +64,16 @@ export interface BrowserAutomationDefinition {
|
|
|
57
64
|
risk: BrowserRiskLevel;
|
|
58
65
|
inputs: Record<string, BrowserAutomationInputDefinition>;
|
|
59
66
|
steps: BrowserAutomationStep[];
|
|
67
|
+
successCriteria?: BrowserAutomationAssertion[];
|
|
68
|
+
outputs?: Record<string, {
|
|
69
|
+
target?: BrowserSemanticTarget;
|
|
70
|
+
field: 'text' | 'value' | 'url' | 'title' | 'href';
|
|
71
|
+
}>;
|
|
60
72
|
}
|
|
61
73
|
export interface BrowserAutomation {
|
|
62
74
|
id: string;
|
|
63
75
|
revision: number;
|
|
76
|
+
verified: boolean;
|
|
64
77
|
status: BrowserAutomationStatus;
|
|
65
78
|
definition: BrowserAutomationDefinition;
|
|
66
79
|
createdAtMs: number;
|
|
@@ -70,6 +83,8 @@ export interface BrowserAutomationRun {
|
|
|
70
83
|
id: string;
|
|
71
84
|
automationId: string;
|
|
72
85
|
automationRevision: number;
|
|
86
|
+
clientRequestId?: string;
|
|
87
|
+
businessOutcome?: 'completed' | 'verified' | 'unknown';
|
|
73
88
|
definition: BrowserAutomationDefinition;
|
|
74
89
|
status: BrowserAutomationRunStatus;
|
|
75
90
|
inputs: Record<string, unknown>;
|
|
@@ -3,7 +3,7 @@ import type { BrowserDriver, BrowserPrimitiveInput } from './browser-driver.js';
|
|
|
3
3
|
type ExtensionProvider = {
|
|
4
4
|
start(): Promise<void>;
|
|
5
5
|
waitForConnection(timeoutMs?: number): Promise<void>;
|
|
6
|
-
send(input: import('@xopcai/browser-control-contract').BrowserActionInput, timeoutMs?: number, visualFallback?: boolean): Promise<import('@xopcai/browser-control-contract').BrowserWireResult>;
|
|
6
|
+
send(input: import('@xopcai/browser-control-contract').BrowserActionInput, timeoutMs?: number, visualFallback?: boolean, signal?: AbortSignal): Promise<import('@xopcai/browser-control-contract').BrowserWireResult>;
|
|
7
7
|
};
|
|
8
8
|
export declare class ExtensionDriver implements BrowserDriver {
|
|
9
9
|
private readonly provider;
|
|
@@ -16,10 +16,10 @@ export declare class ExtensionDriver implements BrowserDriver {
|
|
|
16
16
|
disconnect(): Promise<void>;
|
|
17
17
|
createSession(_sessionId: string): Promise<void>;
|
|
18
18
|
closeSession(sessionId: string): Promise<void>;
|
|
19
|
-
navigate(sessionId: string, input: BrowserNavigateInput): Promise<BrowserControlResult>;
|
|
20
|
-
observe(sessionId: string, input: BrowserObserveInput): Promise<BrowserObservation>;
|
|
21
|
-
perform(sessionId: string, input: BrowserPrimitiveInput): Promise<BrowserControlResult>;
|
|
22
|
-
tabs(sessionId: string, input: BrowserTabsInput): Promise<BrowserControlResult>;
|
|
19
|
+
navigate(sessionId: string, input: BrowserNavigateInput, signal?: AbortSignal): Promise<BrowserControlResult>;
|
|
20
|
+
observe(sessionId: string, input: BrowserObserveInput, signal?: AbortSignal): Promise<BrowserObservation>;
|
|
21
|
+
perform(sessionId: string, input: BrowserPrimitiveInput, signal?: AbortSignal): Promise<BrowserControlResult>;
|
|
22
|
+
tabs(sessionId: string, input: BrowserTabsInput, signal?: AbortSignal): Promise<BrowserControlResult>;
|
|
23
23
|
private send;
|
|
24
24
|
}
|
|
25
25
|
export {};
|
|
@@ -23,36 +23,36 @@ var ExtensionDriver = class {
|
|
|
23
23
|
sessionId
|
|
24
24
|
});
|
|
25
25
|
}
|
|
26
|
-
navigate(sessionId, input) {
|
|
26
|
+
navigate(sessionId, input, signal) {
|
|
27
27
|
return this.send({
|
|
28
28
|
...input,
|
|
29
29
|
sessionId
|
|
30
|
-
});
|
|
30
|
+
}, signal);
|
|
31
31
|
}
|
|
32
|
-
async observe(sessionId, input) {
|
|
32
|
+
async observe(sessionId, input, signal) {
|
|
33
33
|
const result = await this.send({
|
|
34
34
|
...input,
|
|
35
35
|
sessionId
|
|
36
|
-
});
|
|
36
|
+
}, signal);
|
|
37
37
|
if ("error" in result) throw new Error(result.error.message);
|
|
38
38
|
const observation = result.receipt.observation;
|
|
39
39
|
if (!observation) throw new Error("Extension returned no browser observation");
|
|
40
40
|
return observation;
|
|
41
41
|
}
|
|
42
|
-
perform(sessionId, input) {
|
|
42
|
+
perform(sessionId, input, signal) {
|
|
43
43
|
return this.send({
|
|
44
44
|
...input,
|
|
45
45
|
sessionId
|
|
46
|
-
});
|
|
46
|
+
}, signal);
|
|
47
47
|
}
|
|
48
|
-
tabs(sessionId, input) {
|
|
48
|
+
tabs(sessionId, input, signal) {
|
|
49
49
|
return this.send({
|
|
50
50
|
...input,
|
|
51
51
|
sessionId
|
|
52
|
-
});
|
|
52
|
+
}, signal);
|
|
53
53
|
}
|
|
54
|
-
async send(input) {
|
|
55
|
-
return (await this.provider.send(input, this.timeoutMs, this.visualFallback)).result;
|
|
54
|
+
async send(input, signal) {
|
|
55
|
+
return (await this.provider.send(input, this.timeoutMs, this.visualFallback, signal)).result;
|
|
56
56
|
}
|
|
57
57
|
};
|
|
58
58
|
//#endregion
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { browserDom } from "../../../packages/browser-control-contract/src/automation.js";
|
|
2
|
+
import "../../../packages/browser-control-contract/src/index.js";
|
|
1
3
|
import { verifyBrowserExpectation } from "../verification/expectation.js";
|
|
2
4
|
import { randomUUID } from "node:crypto";
|
|
3
5
|
//#region src/browser/drivers/playwright-driver.ts
|
|
@@ -174,6 +176,47 @@ var PlaywrightDriver = class {
|
|
|
174
176
|
try {
|
|
175
177
|
throwIfAborted(signal);
|
|
176
178
|
const state = await this.activeState(sessionId);
|
|
179
|
+
if (input.action === "resolve") {
|
|
180
|
+
const resolved = await state.page.evaluate(browserDom, {
|
|
181
|
+
operation: "resolve",
|
|
182
|
+
target: input.semanticTarget
|
|
183
|
+
});
|
|
184
|
+
if (resolved.matches !== 1) return {
|
|
185
|
+
ok: false,
|
|
186
|
+
error: {
|
|
187
|
+
code: "TARGET_NOT_FOUND",
|
|
188
|
+
message: `Semantic target must match exactly one element; matched ${resolved.matches}.`
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
const handle = await state.page.locator(`[data-xopc-ref="${resolved.node.ref}"]`).elementHandle();
|
|
192
|
+
if (!handle) return targetNotFound(resolved.node.ref);
|
|
193
|
+
state.refs.set(resolved.node.ref, handle);
|
|
194
|
+
state.revision += 1;
|
|
195
|
+
const observation = {
|
|
196
|
+
sessionId,
|
|
197
|
+
tabId: state.id,
|
|
198
|
+
revision: state.revision,
|
|
199
|
+
documentId: state.documentId,
|
|
200
|
+
url: state.page.url(),
|
|
201
|
+
title: await state.page.title(),
|
|
202
|
+
nodes: [resolved.node],
|
|
203
|
+
changes: {
|
|
204
|
+
added: [resolved.node],
|
|
205
|
+
changed: [],
|
|
206
|
+
removed: []
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
return {
|
|
210
|
+
ok: true,
|
|
211
|
+
receipt: {
|
|
212
|
+
action: "resolve",
|
|
213
|
+
risk: "read",
|
|
214
|
+
durationMs: Date.now() - startedAt,
|
|
215
|
+
verified: true,
|
|
216
|
+
observation
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
}
|
|
177
220
|
if ("revision" in input && input.revision !== state.revision) return stale(await this.observe(sessionId, {
|
|
178
221
|
action: "observe",
|
|
179
222
|
visual: "never"
|
|
@@ -188,6 +231,9 @@ var PlaywrightDriver = class {
|
|
|
188
231
|
await handle.fill(input.value, { timeout: this.options.actionTimeoutMs });
|
|
189
232
|
if (input.submit) await handle.press("Enter");
|
|
190
233
|
break;
|
|
234
|
+
case "check":
|
|
235
|
+
await handle.setChecked(input.checked, { timeout: this.options.actionTimeoutMs });
|
|
236
|
+
break;
|
|
191
237
|
case "select":
|
|
192
238
|
await handle.selectOption(input.value);
|
|
193
239
|
break;
|
|
@@ -4,8 +4,7 @@ export type { BrowserDriver, BrowserPrimitiveInput } from './drivers/browser-dri
|
|
|
4
4
|
export { PlaywrightDriver, type PlaywrightDriverOptions } from './drivers/playwright-driver.js';
|
|
5
5
|
export { ExtensionDriver } from './drivers/extension-driver.js';
|
|
6
6
|
export { formatBrowserObservation } from './observation/format.js';
|
|
7
|
-
export { classifyBrowserRisk
|
|
8
|
-
export { createBrowserApproval, consumeBrowserApproval, decideBrowserApproval, listBrowserApprovals, } from './policy/approval-store.js';
|
|
7
|
+
export { classifyBrowserRisk } from './policy/browser-policy.js';
|
|
9
8
|
export { verifyBrowserExpectation } from './verification/expectation.js';
|
|
10
9
|
export { BrowserNotReadyError, buildBrowserSetupDeepLink, checkBrowserReadiness, type BrowserDriverKind, type BrowserNotReadyReason, type BrowserSetupHint, } from './readiness.js';
|
|
11
10
|
export { assertBrowserUrlAllowed, checkPostRedirectUrl, containsApiKeyPattern, isAlwaysBlockedUrl, } from './url-policy.js';
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { assertBrowserUrlAllowed, checkPostRedirectUrl, containsApiKeyPattern, isAlwaysBlockedUrl } from "./url-policy.js";
|
|
2
|
-
import {
|
|
3
|
-
import { browserRiskNeedsApproval, classifyBrowserRisk } from "./policy/browser-policy.js";
|
|
2
|
+
import { classifyBrowserRisk } from "./policy/browser-policy.js";
|
|
4
3
|
import { BrowserRuntime } from "./runtime/browser-runtime.js";
|
|
5
4
|
import { ExtensionDriver } from "./drivers/extension-driver.js";
|
|
6
5
|
import { verifyBrowserExpectation } from "./verification/expectation.js";
|
|
@@ -8,4 +7,4 @@ import { PlaywrightDriver } from "./drivers/playwright-driver.js";
|
|
|
8
7
|
import { createBrowserDriver } from "./drivers/create-driver.js";
|
|
9
8
|
import { formatBrowserObservation } from "./observation/format.js";
|
|
10
9
|
import { BrowserNotReadyError, buildBrowserSetupDeepLink, checkBrowserReadiness } from "./readiness.js";
|
|
11
|
-
export { BrowserNotReadyError, BrowserRuntime, ExtensionDriver, PlaywrightDriver, assertBrowserUrlAllowed,
|
|
10
|
+
export { BrowserNotReadyError, BrowserRuntime, ExtensionDriver, PlaywrightDriver, assertBrowserUrlAllowed, buildBrowserSetupDeepLink, checkBrowserReadiness, checkPostRedirectUrl, classifyBrowserRisk, containsApiKeyPattern, createBrowserDriver, formatBrowserObservation, isAlwaysBlockedUrl, verifyBrowserExpectation };
|
|
@@ -1,3 +1,2 @@
|
|
|
1
1
|
import type { BrowserActionInput, BrowserNode, BrowserRiskLevel } from '@xopcai/browser-control-contract';
|
|
2
2
|
export declare function classifyBrowserRisk(input: BrowserActionInput, target?: BrowserNode): BrowserRiskLevel;
|
|
3
|
-
export declare function browserRiskNeedsApproval(risk: BrowserRiskLevel): boolean;
|
|
@@ -20,8 +20,5 @@ function classifyBrowserRisk(input, target) {
|
|
|
20
20
|
if (EXTERNAL_EFFECT_PATTERN.test(label)) return "external_effect";
|
|
21
21
|
return "draft";
|
|
22
22
|
}
|
|
23
|
-
function browserRiskNeedsApproval(risk) {
|
|
24
|
-
return risk === "external_effect" || risk === "destructive" || risk === "sensitive";
|
|
25
|
-
}
|
|
26
23
|
//#endregion
|
|
27
|
-
export {
|
|
24
|
+
export { classifyBrowserRisk };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export type { CloudBrowserProvider, CloudBrowserProviderConfig } from './types.js';
|
|
2
2
|
export { BrowserbaseProvider } from './browserbase.js';
|
|
3
3
|
export { BrowserUseProvider } from './browser-use.js';
|
|
4
|
-
export {
|
|
4
|
+
export { RealtimeExtensionBrowserProvider } from './realtime-extension.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { BrowserbaseProvider } from "./browserbase.js";
|
|
2
2
|
import { BrowserUseProvider } from "./browser-use.js";
|
|
3
|
-
import {
|
|
4
|
-
export { BrowserUseProvider, BrowserbaseProvider,
|
|
3
|
+
import { RealtimeExtensionBrowserProvider } from "./realtime-extension.js";
|
|
4
|
+
export { BrowserUseProvider, BrowserbaseProvider, RealtimeExtensionBrowserProvider };
|
|
@@ -12,7 +12,7 @@ export declare class RealtimeExtensionBrowserProvider {
|
|
|
12
12
|
constructor(runtime: EndpointToolRuntime, config?: RealtimeExtensionProviderConfig);
|
|
13
13
|
start(): Promise<void>;
|
|
14
14
|
waitForConnection(timeoutMs?: number): Promise<void>;
|
|
15
|
-
send(input: BrowserActionInput, timeoutMs?: number, visualFallback?: boolean): Promise<BrowserWireResult>;
|
|
15
|
+
send(input: BrowserActionInput, timeoutMs?: number, visualFallback?: boolean, signal?: AbortSignal): Promise<BrowserWireResult>;
|
|
16
16
|
isConnected(): boolean;
|
|
17
17
|
getConnectionStatus(): {
|
|
18
18
|
socketConnected: boolean;
|
|
@@ -20,7 +20,7 @@ var RealtimeExtensionBrowserProvider = class {
|
|
|
20
20
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
async send(input, timeoutMs = this.commandTimeout, visualFallback = true) {
|
|
23
|
+
async send(input, timeoutMs = this.commandTimeout, visualFallback = true, signal) {
|
|
24
24
|
const endpoint = this.endpointForInput(input);
|
|
25
25
|
if (!endpoint) {
|
|
26
26
|
if (input.target?.kind === "endpoint") throw new Error(`The Chrome endpoint bound to this Session is offline (${input.target.endpointId}). Reconnect that browser or bind another one.`);
|
|
@@ -31,6 +31,9 @@ var RealtimeExtensionBrowserProvider = class {
|
|
|
31
31
|
if (!tool) throw new Error("Connected browser extension does not support authenticated Realtime browser control. Reload or update the extension.");
|
|
32
32
|
const id = crypto.randomUUID();
|
|
33
33
|
const controller = new AbortController();
|
|
34
|
+
const abort = () => controller.abort();
|
|
35
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
36
|
+
if (signal?.aborted) controller.abort();
|
|
34
37
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
35
38
|
try {
|
|
36
39
|
const value = (await this.runtime.invocations.invoke({
|
|
@@ -53,6 +56,7 @@ var RealtimeExtensionBrowserProvider = class {
|
|
|
53
56
|
};
|
|
54
57
|
} finally {
|
|
55
58
|
clearTimeout(timer);
|
|
59
|
+
signal?.removeEventListener("abort", abort);
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
isConnected() {
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { BrowserRecordingEvent } from '@xopcai/browser-control-contract';
|
|
2
|
+
import type { BrowserAutomationService, BrowserAutomationDefinition } from '../automations/index.js';
|
|
3
|
+
type RecordingRow = {
|
|
4
|
+
recording_id: string;
|
|
5
|
+
principal_id: string;
|
|
6
|
+
state: string;
|
|
7
|
+
ack_seq: number;
|
|
8
|
+
final_seq: number | null;
|
|
9
|
+
automation_id: string | null;
|
|
10
|
+
error: string | null;
|
|
11
|
+
};
|
|
12
|
+
export declare class BrowserRecordingService {
|
|
13
|
+
private readonly automations;
|
|
14
|
+
constructor(automations: BrowserAutomationService);
|
|
15
|
+
get(id: string, principalId: string): RecordingRow | null;
|
|
16
|
+
append(id: string, principalId: string, payload: unknown): {
|
|
17
|
+
ackThrough: number;
|
|
18
|
+
};
|
|
19
|
+
finish(id: string, principalId: string, finalSeq: number): import("../automations/types.js").BrowserAutomation;
|
|
20
|
+
}
|
|
21
|
+
export declare function compileRecording(id: string, events: BrowserRecordingEvent[]): BrowserAutomationDefinition;
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { getSqliteDatabase, runSqliteWriteTransaction } from "../../storage/sqlite/transaction.js";
|
|
2
|
+
import { init_sqlite } from "../../storage/sqlite/index.js";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
//#region src/browser/recordings/service.ts
|
|
5
|
+
init_sqlite();
|
|
6
|
+
const Target = z.object({
|
|
7
|
+
role: z.string().min(1),
|
|
8
|
+
name: z.string().max(240).optional(),
|
|
9
|
+
nameIncludes: z.string().max(240).optional(),
|
|
10
|
+
testId: z.string().max(240).optional(),
|
|
11
|
+
scope: z.object({
|
|
12
|
+
role: z.string(),
|
|
13
|
+
name: z.string().max(240)
|
|
14
|
+
}).strict().optional()
|
|
15
|
+
}).strict();
|
|
16
|
+
const Event = z.object({
|
|
17
|
+
id: z.string().uuid(),
|
|
18
|
+
seq: z.number().int().positive(),
|
|
19
|
+
documentId: z.string().max(200),
|
|
20
|
+
sourceSeq: z.number().int().positive(),
|
|
21
|
+
url: z.string().url().max(4096),
|
|
22
|
+
action: z.enum([
|
|
23
|
+
"navigate",
|
|
24
|
+
"click",
|
|
25
|
+
"fill",
|
|
26
|
+
"select",
|
|
27
|
+
"check",
|
|
28
|
+
"press",
|
|
29
|
+
"checkpoint",
|
|
30
|
+
"unsupported"
|
|
31
|
+
]),
|
|
32
|
+
target: Target.optional(),
|
|
33
|
+
value: z.string().max(4096).optional(),
|
|
34
|
+
checked: z.boolean().optional(),
|
|
35
|
+
key: z.enum(["Enter", "Escape"]).optional()
|
|
36
|
+
}).strict();
|
|
37
|
+
const Batch = z.array(Event).min(1).max(100);
|
|
38
|
+
var BrowserRecordingService = class {
|
|
39
|
+
constructor(automations) {
|
|
40
|
+
this.automations = automations;
|
|
41
|
+
}
|
|
42
|
+
get(id, principalId) {
|
|
43
|
+
return getSqliteDatabase().prepare("SELECT * FROM browser_recordings WHERE recording_id=? AND principal_id=?").get(id, principalId) ?? null;
|
|
44
|
+
}
|
|
45
|
+
append(id, principalId, payload) {
|
|
46
|
+
z.string().uuid().parse(id);
|
|
47
|
+
const events = Batch.parse(payload);
|
|
48
|
+
return runSqliteWriteTransaction((db) => {
|
|
49
|
+
const now = Date.now();
|
|
50
|
+
db.prepare("INSERT OR IGNORE INTO browser_recordings (recording_id, principal_id, state, created_at_ms, updated_at_ms) VALUES (?, ?, 'recording', ?, ?)").run(id, principalId, now, now);
|
|
51
|
+
const row = this.get(id, principalId);
|
|
52
|
+
if (!row) throw new Error("Recording belongs to another device.");
|
|
53
|
+
if (Math.max(row.ack_seq, events.at(-1).seq) > 1e4) throw new Error("Recording event budget exceeded.");
|
|
54
|
+
let bytes = db.prepare("SELECT COALESCE(SUM(length(CAST(payload_json AS BLOB))),0) AS bytes FROM browser_recording_events WHERE recording_id=?").get(id).bytes;
|
|
55
|
+
for (const event of events) {
|
|
56
|
+
const prior = db.prepare("SELECT payload_json FROM browser_recording_events WHERE recording_id=? AND seq=?").get(id, event.seq);
|
|
57
|
+
const serialized = JSON.stringify(event);
|
|
58
|
+
if (prior) {
|
|
59
|
+
if (prior.payload_json !== serialized) throw new Error("Recording event conflict.");
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
bytes += Buffer.byteLength(serialized);
|
|
63
|
+
if (bytes > 20 * 1024 * 1024) throw new Error("Recording storage budget exceeded.");
|
|
64
|
+
if (row.state === "saved") throw new Error("Recording is already finished.");
|
|
65
|
+
if (event.seq !== this.get(id, principalId).ack_seq + 1) throw new Error("Recording event sequence has a gap.");
|
|
66
|
+
if (!["http:", "https:"].includes(new URL(event.url).protocol)) throw new Error("Only HTTP pages can be recorded.");
|
|
67
|
+
db.prepare("INSERT INTO browser_recording_events (recording_id, seq, event_id, payload_json) VALUES (?, ?, ?, ?)").run(id, event.seq, event.id, serialized);
|
|
68
|
+
db.prepare("UPDATE browser_recordings SET ack_seq=?, updated_at_ms=? WHERE recording_id=?").run(event.seq, now, id);
|
|
69
|
+
}
|
|
70
|
+
return { ackThrough: this.get(id, principalId).ack_seq };
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
finish(id, principalId, finalSeq) {
|
|
74
|
+
z.number().int().min(1).max(1e4).parse(finalSeq);
|
|
75
|
+
const row = this.get(id, principalId);
|
|
76
|
+
if (!row) throw new Error("Recording not found.");
|
|
77
|
+
if (row.state === "saved") {
|
|
78
|
+
if (row.final_seq !== finalSeq) throw new Error("Recording final sequence conflicts with the saved recording.");
|
|
79
|
+
return this.automations.get(row.automation_id);
|
|
80
|
+
}
|
|
81
|
+
if (row.ack_seq !== finalSeq) throw new Error("Recording is incomplete. Waiting for all events.");
|
|
82
|
+
const events = getSqliteDatabase().prepare("SELECT payload_json FROM browser_recording_events WHERE recording_id=? ORDER BY seq").all(id).map((item) => JSON.parse(item.payload_json));
|
|
83
|
+
const sources = /* @__PURE__ */ new Map();
|
|
84
|
+
for (const event of events) {
|
|
85
|
+
if (event.sourceSeq !== (sources.get(event.documentId) ?? 0) + 1) throw new Error("Recording is incomplete: document events are missing.");
|
|
86
|
+
sources.set(event.documentId, event.sourceSeq);
|
|
87
|
+
}
|
|
88
|
+
for (const documentId of sources.keys()) if (events.filter((event) => event.documentId === documentId).at(-1)?.action !== "checkpoint") throw new Error("Recording is incomplete: a document did not confirm its final events.");
|
|
89
|
+
const definition = compileRecording(id, events);
|
|
90
|
+
return runSqliteWriteTransaction((db) => {
|
|
91
|
+
const automation = this.automations.save({ definition });
|
|
92
|
+
db.prepare("UPDATE browser_recordings SET state='saved', final_seq=?, automation_id=?, updated_at_ms=? WHERE recording_id=?").run(finalSeq, automation.id, Date.now(), id);
|
|
93
|
+
return automation;
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
function compileRecording(id, events) {
|
|
98
|
+
const steps = [];
|
|
99
|
+
const inputs = {};
|
|
100
|
+
for (const event of events) {
|
|
101
|
+
if (event.action === "checkpoint") continue;
|
|
102
|
+
if (event.action === "unsupported") throw new Error(event.value || "Recording contains an unsupported action.");
|
|
103
|
+
if (event.action === "navigate") {
|
|
104
|
+
if (steps.length && ["click", "press"].includes(steps.at(-1).action)) steps.push({
|
|
105
|
+
action: "wait",
|
|
106
|
+
condition: "page_idle"
|
|
107
|
+
});
|
|
108
|
+
else if (!steps.length || steps.at(-1)?.action !== "wait") steps.push({
|
|
109
|
+
action: "navigate",
|
|
110
|
+
url: event.url
|
|
111
|
+
});
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (!event.target) throw new Error("Recording contains an unsupported target.");
|
|
115
|
+
if (event.action === "fill" || event.action === "select") {
|
|
116
|
+
const key = `value${Object.keys(inputs).length + 1}`;
|
|
117
|
+
inputs[key] = {
|
|
118
|
+
type: "string",
|
|
119
|
+
default: event.value ?? "",
|
|
120
|
+
description: event.target.name || key
|
|
121
|
+
};
|
|
122
|
+
steps.push({
|
|
123
|
+
action: event.action,
|
|
124
|
+
target: event.target,
|
|
125
|
+
value: `\${input.${key}}`
|
|
126
|
+
});
|
|
127
|
+
} else if (event.action === "check") {
|
|
128
|
+
if (event.checked === void 0) throw new Error("Recording checkbox state is missing.");
|
|
129
|
+
steps.push({
|
|
130
|
+
action: "check",
|
|
131
|
+
target: event.target,
|
|
132
|
+
checked: event.checked
|
|
133
|
+
});
|
|
134
|
+
} else if (event.action === "press") {
|
|
135
|
+
if (!event.key) throw new Error("Recording key is missing.");
|
|
136
|
+
steps.push({
|
|
137
|
+
action: "press",
|
|
138
|
+
target: event.target,
|
|
139
|
+
key: event.key
|
|
140
|
+
});
|
|
141
|
+
} else steps.push({
|
|
142
|
+
action: "click",
|
|
143
|
+
target: event.target
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
if (!steps.length || steps.length > 100) throw new Error("Recording must contain between 1 and 100 executable steps.");
|
|
147
|
+
const first = events[0];
|
|
148
|
+
return {
|
|
149
|
+
id: `recording-${id}`,
|
|
150
|
+
name: events.find((event) => event.target?.name)?.target?.name || new URL(first.url).hostname,
|
|
151
|
+
description: "Recorded browser workflow",
|
|
152
|
+
allowedDomains: [...new Set(events.map((event) => new URL(event.url).hostname))],
|
|
153
|
+
risk: steps.some((step) => ["click", "press"].includes(step.action)) ? "external_effect" : "draft",
|
|
154
|
+
inputs,
|
|
155
|
+
steps,
|
|
156
|
+
outputs: { url: { field: "url" } }
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
//#endregion
|
|
160
|
+
export { BrowserRecordingService, compileRecording };
|
|
@@ -18,7 +18,7 @@ export declare class BrowserRuntime {
|
|
|
18
18
|
closeTaskSession(taskKey: string): Promise<void>;
|
|
19
19
|
private dispatch;
|
|
20
20
|
private executeSequence;
|
|
21
|
-
private
|
|
21
|
+
private actionDenied;
|
|
22
22
|
private validateUrl;
|
|
23
23
|
private validateRedirect;
|
|
24
24
|
private blockedRedirect;
|