@google/adk 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/a2a/a2a_remote_agent.js +17 -2
- package/dist/cjs/a2a/a2a_remote_agent_utils.js +8 -4
- package/dist/cjs/a2a/agent_card.js +5 -81
- package/dist/cjs/a2a/agent_to_a2a.js +7 -0
- package/dist/cjs/a2a/event_converter_utils.js +10 -3
- package/dist/cjs/agents/agent_engine_client.js +282 -55
- package/dist/cjs/agents/base_agent.js +111 -68
- package/dist/cjs/agents/functions.js +62 -85
- package/dist/cjs/agents/instructions.js +7 -6
- package/dist/cjs/agents/invocation_context.js +2 -2
- package/dist/cjs/agents/llm_agent.js +105 -95
- package/dist/cjs/agents/processors/content_processor_utils.js +42 -15
- package/dist/cjs/agents/processors/content_request_processor.js +28 -11
- package/dist/cjs/agents/processors/context_compactor_request_processor.js +5 -1
- package/dist/cjs/agents/processors/instructions_llm_request_processor.js +1 -1
- package/dist/cjs/agents/processors/nl_planning_processor.js +2 -1
- package/dist/cjs/artifacts/file_artifact_service.js +1 -1
- package/dist/cjs/cli/cli_eval.js +884 -17
- package/dist/cjs/cli/utils/cleanup.js +84 -0
- package/dist/cjs/cli/utils/envs.js +33 -2
- package/dist/cjs/cli/utils/evals.js +75 -8
- package/dist/cjs/cli/utils/logs.js +128 -9
- package/dist/cjs/cli/utils/path_validation.js +83 -0
- package/dist/cjs/code_executors/container_code_executor.js +158 -36
- package/dist/cjs/common.js +50 -12
- package/dist/cjs/context/compaction_utils.js +19 -0
- package/dist/cjs/events/event.js +7 -3
- package/dist/cjs/features/feature_registry.js +5 -0
- package/dist/cjs/index.js +29 -14
- package/dist/cjs/models/google_llm.js +33 -2
- package/dist/cjs/models/interactions_utils.js +3 -0
- package/dist/cjs/plugins/base_plugin.js +45 -21
- package/dist/cjs/plugins/plugin_manager.js +27 -0
- package/dist/cjs/runner/runner.js +306 -189
- package/dist/cjs/sessions/base_session_service.js +1 -4
- package/dist/cjs/sessions/database_session_service.js +191 -19
- package/dist/cjs/sessions/db/operations.js +70 -1
- package/dist/cjs/sessions/db/schema.js +4 -0
- package/dist/cjs/sessions/in_memory_session_service.js +2 -2
- package/dist/cjs/sessions/vertex_ai_session_service.js +11 -3
- package/dist/cjs/skills/loader.js +29 -28
- package/dist/cjs/telemetry/api/instrumentation.js +632 -0
- package/dist/cjs/telemetry/api/internal/common/accumulators.js +169 -0
- package/dist/cjs/telemetry/api/internal/common/attributes.js +131 -0
- package/dist/cjs/telemetry/api/internal/common/config.js +159 -0
- package/dist/cjs/telemetry/api/internal/common/error.js +98 -0
- package/dist/cjs/telemetry/api/internal/common/finish_reason.js +63 -0
- package/dist/cjs/telemetry/api/internal/common/genai_messages.js +217 -0
- package/dist/cjs/telemetry/api/internal/common/http_exchange.js +169 -0
- package/dist/cjs/telemetry/api/internal/common/provider_name.js +87 -0
- package/dist/cjs/telemetry/api/internal/common/redaction.js +148 -0
- package/dist/cjs/telemetry/api/internal/common/serialization.js +58 -0
- package/dist/cjs/telemetry/api/internal/common/token_usage.js +118 -0
- package/dist/cjs/telemetry/api/internal/logs/inference.js +50 -0
- package/dist/cjs/telemetry/api/internal/logs/logger.js +46 -0
- package/dist/cjs/telemetry/api/internal/logs/mcp.js +141 -0
- package/dist/cjs/telemetry/api/internal/metrics/agent.js +128 -0
- package/dist/cjs/telemetry/api/internal/metrics/inference_start_hook.js +56 -0
- package/dist/cjs/telemetry/api/internal/metrics/meters.js +189 -0
- package/dist/cjs/telemetry/api/internal/metrics/tool.js +147 -0
- package/dist/cjs/telemetry/api/internal/metrics/workflow.js +132 -0
- package/dist/cjs/telemetry/api/internal/trace/agent.js +52 -0
- package/dist/cjs/telemetry/api/internal/trace/compaction.js +78 -0
- package/dist/cjs/telemetry/api/internal/trace/inference.js +132 -0
- package/dist/cjs/telemetry/api/internal/trace/live.js +52 -0
- package/dist/cjs/telemetry/api/internal/trace/node.js +73 -0
- package/dist/cjs/telemetry/api/internal/trace/propagation.js +67 -0
- package/dist/cjs/telemetry/api/internal/trace/tool.js +122 -0
- package/dist/cjs/telemetry/api/internal/trace/tracer.js +77 -0
- package/dist/cjs/telemetry/api/internal/trace/workflow.js +77 -0
- package/dist/cjs/telemetry/export/agent_engine.js +90 -0
- package/dist/cjs/telemetry/export/agent_runtime.js +73 -0
- package/dist/cjs/telemetry/export/google_cloud.js +763 -0
- package/dist/cjs/telemetry/export/request_driven_metric_reader.js +303 -0
- package/dist/cjs/telemetry/{setup.js → export/setup.js} +72 -5
- package/dist/cjs/tools/agent_tool.js +9 -1
- package/dist/cjs/tools/application_integration_tool/application_integration_toolset.js +22 -34
- package/dist/cjs/tools/application_integration_tool/clients/connections_client.js +60 -20
- package/dist/cjs/tools/application_integration_tool/clients/integration_client.js +26 -10
- package/dist/cjs/tools/application_integration_tool/integration_connector_tool.js +14 -23
- package/dist/cjs/tools/base_toolset.js +1 -1
- package/dist/cjs/tools/computer_use/base_computer.js +125 -0
- package/dist/cjs/tools/computer_use/computer_use_actions.js +502 -0
- package/dist/cjs/tools/computer_use/computer_use_tool.js +244 -0
- package/dist/cjs/tools/computer_use/computer_use_toolset.js +360 -0
- package/dist/cjs/tools/function_tool.js +15 -5
- package/dist/cjs/tools/google_api_tool/google_api_toolset.js +3 -2
- package/dist/cjs/tools/google_api_tool/googleapi_to_openapi_converter.js +3 -1
- package/dist/cjs/tools/load_artifacts_tool.js +15 -5
- package/dist/cjs/tools/load_memory_tool.js +2 -5
- package/dist/cjs/tools/mcp/mcp_session_manager.js +23 -3
- package/dist/cjs/tools/mcp/mcp_tool.js +7 -3
- package/dist/cjs/tools/mcp/mcp_toolset.js +18 -6
- package/dist/cjs/tools/openapi_tool/auth/auth_helpers.js +30 -29
- package/dist/cjs/tools/openapi_tool/auth/credential_exchangers/oauth2_exchanger.js +2 -2
- package/dist/cjs/tools/openapi_tool/rest_api_tool.js +45 -14
- package/dist/cjs/tools/skill/load_skill_resource_tool.js +7 -0
- package/dist/cjs/tools/skill/load_skill_tool.js +3 -0
- package/dist/cjs/tools/skill/run_skill_script_tool.js +12 -2
- package/dist/cjs/tools/toolbox_toolset.js +47 -28
- package/dist/cjs/utils/async_queue.js +37 -3
- package/dist/cjs/utils/client_labels.js +9 -11
- package/dist/cjs/utils/env_aware_utils.js +13 -7
- package/dist/cjs/utils/error_utils.js +40 -2
- package/dist/cjs/utils/failover_utils.js +10 -9
- package/dist/cjs/utils/gemini_schema_util.js +125 -12
- package/dist/cjs/utils/id_provider.js +63 -0
- package/dist/cjs/utils/model_name.js +15 -0
- package/dist/cjs/utils/simple_zod_to_json.js +13 -31
- package/dist/cjs/utils/ssrf_guard.js +46 -16
- package/dist/cjs/version.js +1 -1
- package/dist/cjs/workflow/index.js +2 -0
- package/dist/cjs/workflow/node_runner.js +24 -80
- package/dist/cjs/workflow/nodes/parallel_worker.js +18 -7
- package/dist/cjs/workflow/run_llm_agent_as_node.js +19 -14
- package/dist/cjs/workflow/utils/retry_utils.js +2 -2
- package/dist/cjs/workflow/workflow.js +12 -12
- package/dist/esm/a2a/a2a_remote_agent.js +23 -3
- package/dist/esm/a2a/a2a_remote_agent_utils.js +8 -4
- package/dist/esm/a2a/agent_card.js +5 -83
- package/dist/esm/a2a/agent_to_a2a.js +7 -0
- package/dist/esm/a2a/event_converter_utils.js +10 -3
- package/dist/esm/agents/agent_engine_client.js +293 -55
- package/dist/esm/agents/base_agent.js +111 -72
- package/dist/esm/agents/functions.js +67 -89
- package/dist/esm/agents/instructions.js +7 -6
- package/dist/esm/agents/invocation_context.js +2 -2
- package/dist/esm/agents/llm_agent.js +111 -100
- package/dist/esm/agents/processors/content_processor_utils.js +42 -15
- package/dist/esm/agents/processors/content_request_processor.js +28 -11
- package/dist/esm/agents/processors/context_compactor_request_processor.js +5 -1
- package/dist/esm/agents/processors/instructions_llm_request_processor.js +1 -1
- package/dist/esm/agents/processors/nl_planning_processor.js +2 -1
- package/dist/esm/artifacts/file_artifact_service.js +1 -1
- package/dist/esm/cli/cli_eval.js +880 -17
- package/dist/esm/cli/utils/cleanup.js +56 -0
- package/dist/esm/cli/utils/envs.js +33 -2
- package/dist/esm/cli/utils/evals.js +74 -8
- package/dist/esm/cli/utils/logs.js +128 -9
- package/dist/esm/cli/utils/path_validation.js +45 -0
- package/dist/esm/code_executors/container_code_executor.js +158 -36
- package/dist/esm/common.js +49 -6
- package/dist/esm/context/compaction_utils.js +19 -0
- package/dist/esm/events/event.js +7 -3
- package/dist/esm/features/feature_registry.js +5 -0
- package/dist/esm/index.js +27 -15
- package/dist/esm/models/google_llm.js +33 -2
- package/dist/esm/models/interactions_utils.js +3 -0
- package/dist/esm/plugins/base_plugin.js +45 -21
- package/dist/esm/plugins/plugin_manager.js +27 -0
- package/dist/esm/runner/runner.js +310 -192
- package/dist/esm/sessions/base_session_service.js +1 -4
- package/dist/esm/sessions/database_session_service.js +191 -19
- package/dist/esm/sessions/db/operations.js +70 -1
- package/dist/esm/sessions/db/schema.js +4 -0
- package/dist/esm/sessions/in_memory_session_service.js +2 -2
- package/dist/esm/sessions/vertex_ai_session_service.js +11 -3
- package/dist/esm/skills/loader.js +29 -28
- package/dist/esm/telemetry/api/instrumentation.js +655 -0
- package/dist/esm/telemetry/api/internal/common/accumulators.js +139 -0
- package/dist/esm/telemetry/api/internal/common/attributes.js +94 -0
- package/dist/esm/telemetry/api/internal/common/config.js +128 -0
- package/dist/esm/telemetry/api/internal/common/error.js +69 -0
- package/dist/esm/telemetry/api/internal/common/finish_reason.js +35 -0
- package/dist/esm/telemetry/api/internal/common/genai_messages.js +188 -0
- package/dist/esm/telemetry/api/internal/common/http_exchange.js +142 -0
- package/dist/esm/telemetry/api/internal/common/provider_name.js +59 -0
- package/dist/esm/telemetry/api/internal/common/redaction.js +121 -0
- package/dist/esm/telemetry/api/internal/common/serialization.js +30 -0
- package/dist/esm/telemetry/api/internal/common/token_usage.js +85 -0
- package/dist/esm/telemetry/api/internal/logs/inference.js +23 -0
- package/dist/esm/telemetry/api/internal/logs/logger.js +19 -0
- package/dist/esm/telemetry/api/internal/logs/mcp.js +113 -0
- package/dist/esm/telemetry/api/internal/metrics/agent.js +107 -0
- package/dist/esm/telemetry/api/internal/metrics/inference_start_hook.js +29 -0
- package/dist/esm/telemetry/api/internal/metrics/meters.js +165 -0
- package/dist/esm/telemetry/api/internal/metrics/tool.js +119 -0
- package/dist/esm/telemetry/api/internal/metrics/workflow.js +112 -0
- package/dist/esm/telemetry/api/internal/trace/agent.js +25 -0
- package/dist/esm/telemetry/api/internal/trace/compaction.js +50 -0
- package/dist/esm/telemetry/api/internal/trace/inference.js +107 -0
- package/dist/esm/telemetry/api/internal/trace/live.js +28 -0
- package/dist/esm/telemetry/api/internal/trace/node.js +44 -0
- package/dist/esm/telemetry/api/internal/trace/propagation.js +39 -0
- package/dist/esm/telemetry/api/internal/trace/tool.js +96 -0
- package/dist/esm/telemetry/api/internal/trace/tracer.js +52 -0
- package/dist/esm/telemetry/api/internal/trace/workflow.js +54 -0
- package/dist/esm/telemetry/export/agent_engine.js +67 -0
- package/dist/esm/telemetry/export/agent_runtime.js +46 -0
- package/dist/esm/telemetry/export/google_cloud.js +733 -0
- package/dist/esm/telemetry/export/request_driven_metric_reader.js +276 -0
- package/dist/esm/telemetry/export/setup.js +160 -0
- package/dist/esm/tools/agent_tool.js +9 -1
- package/dist/esm/tools/application_integration_tool/application_integration_toolset.js +23 -35
- package/dist/esm/tools/application_integration_tool/clients/connections_client.js +64 -21
- package/dist/esm/tools/application_integration_tool/clients/integration_client.js +26 -10
- package/dist/esm/tools/application_integration_tool/integration_connector_tool.js +14 -23
- package/dist/esm/tools/base_toolset.js +1 -1
- package/dist/esm/tools/computer_use/base_computer.js +99 -0
- package/dist/esm/tools/computer_use/computer_use_actions.js +477 -0
- package/dist/esm/tools/computer_use/computer_use_tool.js +217 -0
- package/dist/esm/tools/computer_use/computer_use_toolset.js +344 -0
- package/dist/esm/tools/function_tool.js +15 -5
- package/dist/esm/tools/google_api_tool/google_api_toolset.js +3 -2
- package/dist/esm/tools/google_api_tool/googleapi_to_openapi_converter.js +3 -1
- package/dist/esm/tools/load_artifacts_tool.js +15 -5
- package/dist/esm/tools/load_memory_tool.js +2 -5
- package/dist/esm/tools/mcp/mcp_session_manager.js +23 -3
- package/dist/esm/tools/mcp/mcp_tool.js +7 -3
- package/dist/esm/tools/mcp/mcp_toolset.js +18 -6
- package/dist/esm/tools/openapi_tool/auth/auth_helpers.js +27 -26
- package/dist/esm/tools/openapi_tool/auth/credential_exchangers/oauth2_exchanger.js +2 -2
- package/dist/esm/tools/openapi_tool/rest_api_tool.js +45 -14
- package/dist/esm/tools/skill/load_skill_resource_tool.js +7 -0
- package/dist/esm/tools/skill/load_skill_tool.js +3 -0
- package/dist/esm/tools/skill/run_skill_script_tool.js +12 -2
- package/dist/esm/tools/toolbox_toolset.js +47 -28
- package/dist/esm/utils/async_queue.js +37 -3
- package/dist/esm/utils/client_labels.js +9 -11
- package/dist/esm/utils/env_aware_utils.js +12 -7
- package/dist/esm/utils/error_utils.js +37 -1
- package/dist/esm/utils/failover_utils.js +10 -9
- package/dist/esm/utils/gemini_schema_util.js +124 -11
- package/dist/esm/utils/id_provider.js +33 -0
- package/dist/esm/utils/model_name.js +14 -0
- package/dist/esm/utils/simple_zod_to_json.js +13 -31
- package/dist/esm/utils/ssrf_guard.js +55 -15
- package/dist/esm/version.js +1 -1
- package/dist/esm/workflow/index.js +2 -1
- package/dist/esm/workflow/node_runner.js +26 -80
- package/dist/esm/workflow/nodes/parallel_worker.js +16 -6
- package/dist/esm/workflow/run_llm_agent_as_node.js +19 -14
- package/dist/esm/workflow/utils/retry_utils.js +2 -2
- package/dist/esm/workflow/workflow.js +12 -12
- package/dist/types/a2a/a2a_remote_agent.d.ts +6 -0
- package/dist/types/a2a/a2a_remote_agent_utils.d.ts +3 -2
- package/dist/types/agents/agent_engine_client.d.ts +121 -3
- package/dist/types/agents/base_agent.d.ts +2 -0
- package/dist/types/agents/processors/content_processor_utils.d.ts +9 -9
- package/dist/types/agents/run_config.d.ts +14 -0
- package/dist/types/cli/cli_eval.d.ts +172 -9
- package/dist/types/cli/utils/cleanup.d.ts +18 -0
- package/dist/types/cli/utils/envs.d.ts +13 -3
- package/dist/types/cli/utils/evals.d.ts +64 -1
- package/dist/types/cli/utils/logs.d.ts +19 -2
- package/dist/types/cli/utils/path_validation.d.ts +30 -0
- package/dist/types/code_executors/container_code_executor.d.ts +49 -8
- package/dist/types/common.d.ts +16 -9
- package/dist/types/context/compaction_utils.d.ts +13 -1
- package/dist/types/events/event.d.ts +3 -0
- package/dist/types/features/feature_registry.d.ts +1 -0
- package/dist/types/index.d.ts +9 -6
- package/dist/types/plugins/base_plugin.d.ts +50 -21
- package/dist/types/plugins/plugin_manager.d.ts +13 -0
- package/dist/types/runner/runner.d.ts +11 -0
- package/dist/types/sessions/base_session_service.d.ts +0 -2
- package/dist/types/sessions/db/operations.d.ts +2 -1
- package/dist/types/sessions/db/schema.d.ts +5 -3
- package/dist/types/telemetry/api/instrumentation.d.ts +259 -0
- package/dist/types/telemetry/api/internal/common/accumulators.d.ts +91 -0
- package/dist/types/telemetry/api/internal/common/attributes.d.ts +90 -0
- package/dist/types/telemetry/api/internal/common/config.d.ts +61 -0
- package/dist/types/telemetry/api/internal/common/error.d.ts +29 -0
- package/dist/types/telemetry/api/internal/common/finish_reason.d.ts +18 -0
- package/dist/types/telemetry/api/internal/common/genai_messages.d.ts +26 -0
- package/dist/types/telemetry/api/internal/common/http_exchange.d.ts +50 -0
- package/dist/types/telemetry/api/internal/common/provider_name.d.ts +16 -0
- package/dist/types/telemetry/api/internal/common/redaction.d.ts +14 -0
- package/dist/types/telemetry/api/internal/common/serialization.d.ts +18 -0
- package/dist/types/telemetry/api/internal/common/token_usage.d.ts +47 -0
- package/dist/types/telemetry/api/internal/logs/inference.d.ts +15 -0
- package/dist/types/telemetry/api/internal/logs/logger.d.ts +13 -0
- package/dist/types/telemetry/api/internal/logs/mcp.d.ts +31 -0
- package/dist/types/telemetry/api/internal/metrics/agent.d.ts +28 -0
- package/dist/types/telemetry/api/internal/metrics/inference_start_hook.d.ts +12 -0
- package/dist/types/telemetry/api/internal/metrics/meters.d.ts +57 -0
- package/dist/types/telemetry/api/internal/metrics/tool.d.ts +44 -0
- package/dist/types/telemetry/api/internal/metrics/workflow.d.ts +32 -0
- package/dist/types/telemetry/api/internal/trace/agent.d.ts +14 -0
- package/dist/types/telemetry/api/internal/trace/compaction.d.ts +20 -0
- package/dist/types/telemetry/api/internal/trace/inference.d.ts +31 -0
- package/dist/types/telemetry/api/internal/trace/live.d.ts +21 -0
- package/dist/types/telemetry/api/internal/trace/node.d.ts +22 -0
- package/dist/types/telemetry/api/internal/trace/propagation.d.ts +22 -0
- package/dist/types/telemetry/api/internal/trace/tool.d.ts +57 -0
- package/dist/types/telemetry/api/internal/trace/tracer.d.ts +31 -0
- package/dist/types/telemetry/api/internal/trace/workflow.d.ts +21 -0
- package/dist/types/telemetry/export/agent_engine.d.ts +23 -0
- package/dist/types/telemetry/export/agent_runtime.d.ts +40 -0
- package/dist/types/telemetry/export/google_cloud.d.ts +189 -0
- package/dist/types/telemetry/export/request_driven_metric_reader.d.ts +144 -0
- package/dist/types/telemetry/{setup.d.ts → export/setup.d.ts} +3 -0
- package/dist/types/tools/agent_tool.d.ts +1 -1
- package/dist/types/tools/application_integration_tool/clients/connections_client.d.ts +13 -4
- package/dist/types/tools/base_toolset.d.ts +1 -1
- package/dist/types/tools/computer_use/base_computer.d.ts +281 -0
- package/dist/types/tools/computer_use/computer_use_actions.d.ts +53 -0
- package/dist/types/tools/computer_use/computer_use_tool.d.ts +108 -0
- package/dist/types/tools/computer_use/computer_use_toolset.d.ts +132 -0
- package/dist/types/tools/google_api_tool/google_api_toolset.d.ts +2 -0
- package/dist/types/tools/load_artifacts_tool.d.ts +5 -1
- package/dist/types/tools/load_memory_tool.d.ts +13 -7
- package/dist/types/tools/mcp/mcp_session_manager.d.ts +16 -0
- package/dist/types/tools/openapi_tool/auth/auth_helpers.d.ts +8 -8
- package/dist/types/tools/openapi_tool/openapi_spec_parser/operation_parser.d.ts +7 -3
- package/dist/types/tools/toolbox_toolset.d.ts +15 -20
- package/dist/types/utils/async_queue.d.ts +14 -0
- package/dist/types/utils/env_aware_utils.d.ts +11 -0
- package/dist/types/utils/error_utils.d.ts +4 -0
- package/dist/types/utils/gemini_schema_util.d.ts +27 -6
- package/dist/types/utils/id_provider.d.ts +43 -0
- package/dist/types/utils/model_name.d.ts +8 -0
- package/dist/types/utils/ssrf_guard.d.ts +2 -0
- package/dist/types/version.d.ts +1 -1
- package/dist/types/workflow/index.d.ts +1 -1
- package/dist/types/workflow/nodes/parallel_worker.d.ts +8 -1
- package/dist/web/index_web.js +86 -88
- package/dist/web/index_web.js.map +4 -4
- package/package.json +13 -15
- package/dist/cjs/telemetry/google_cloud.js +0 -116
- package/dist/cjs/telemetry/tracing.js +0 -393
- package/dist/esm/telemetry/google_cloud.js +0 -99
- package/dist/esm/telemetry/setup.js +0 -88
- package/dist/esm/telemetry/tracing.js +0 -361
- package/dist/types/telemetry/google_cloud.d.ts +0 -9
- package/dist/types/telemetry/tracing.d.ts +0 -143
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
import {createRequire as topLevelCreateRequire} from 'module';
|
|
2
|
+
const require = topLevelCreateRequire(import.meta.url);
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
5
|
+
/**
|
|
6
|
+
* @license
|
|
7
|
+
* Copyright 2026 Google LLC
|
|
8
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
9
|
+
*/
|
|
10
|
+
import { Type } from "@google/genai";
|
|
11
|
+
import { isBrowser } from "../../utils/env_aware_utils.js";
|
|
12
|
+
import { logger } from "../../utils/logger.js";
|
|
13
|
+
import {
|
|
14
|
+
isHttpUrl,
|
|
15
|
+
isIpAddress,
|
|
16
|
+
isNonGlobalAddress,
|
|
17
|
+
normalizeHost,
|
|
18
|
+
resolveHostAddresses
|
|
19
|
+
} from "../../utils/ssrf_guard.js";
|
|
20
|
+
const LEGACY_ACTION_NAMES = [
|
|
21
|
+
"open_web_browser",
|
|
22
|
+
"click_at",
|
|
23
|
+
"hover_at",
|
|
24
|
+
"type_text_at",
|
|
25
|
+
"scroll_document",
|
|
26
|
+
"scroll_at",
|
|
27
|
+
"wait",
|
|
28
|
+
"go_back",
|
|
29
|
+
"go_forward",
|
|
30
|
+
"search",
|
|
31
|
+
"navigate",
|
|
32
|
+
"key_combination",
|
|
33
|
+
"drag_and_drop",
|
|
34
|
+
"current_state"
|
|
35
|
+
];
|
|
36
|
+
const OPTIONAL_MOUSE_ACTIONS = [
|
|
37
|
+
"double_click",
|
|
38
|
+
"triple_click",
|
|
39
|
+
"middle_click",
|
|
40
|
+
"right_click",
|
|
41
|
+
"mouse_down",
|
|
42
|
+
"mouse_up"
|
|
43
|
+
];
|
|
44
|
+
const OPTIONAL_KEY_ACTIONS = ["key_down", "key_up"];
|
|
45
|
+
const MOUSE_METHODS = {
|
|
46
|
+
double_click: "doubleClick",
|
|
47
|
+
triple_click: "tripleClick",
|
|
48
|
+
middle_click: "middleClick",
|
|
49
|
+
right_click: "rightClick",
|
|
50
|
+
mouse_down: "mouseDown",
|
|
51
|
+
mouse_up: "mouseUp"
|
|
52
|
+
};
|
|
53
|
+
const KEY_METHODS = {
|
|
54
|
+
key_down: "keyDown",
|
|
55
|
+
key_up: "keyUp"
|
|
56
|
+
};
|
|
57
|
+
const SCROLL_DIRECTIONS = [
|
|
58
|
+
"up",
|
|
59
|
+
"down",
|
|
60
|
+
"left",
|
|
61
|
+
"right"
|
|
62
|
+
];
|
|
63
|
+
const BLOCKED_HOSTNAMES = /* @__PURE__ */ new Set(["localhost", "metadata", "metadata.goog"]);
|
|
64
|
+
const BLOCKED_HOSTNAME_SUFFIXES = [
|
|
65
|
+
".localhost",
|
|
66
|
+
".local",
|
|
67
|
+
".internal",
|
|
68
|
+
".metadata.goog"
|
|
69
|
+
];
|
|
70
|
+
const URL_AUTHORITY = /^[a-z][a-z0-9+.-]*:\/\/([^/?#]*)/i;
|
|
71
|
+
const URL_REFUSED_ERROR = "navigate refused: url must be http(s) and must not target a private or link-local address.";
|
|
72
|
+
const INTEGER = { type: Type.INTEGER };
|
|
73
|
+
const NUMBER = { type: Type.NUMBER };
|
|
74
|
+
const STRING = { type: Type.STRING };
|
|
75
|
+
const BOOLEAN = { type: Type.BOOLEAN };
|
|
76
|
+
const STRING_ARRAY = { type: Type.ARRAY, items: { type: Type.STRING } };
|
|
77
|
+
const DIRECTION = { type: Type.STRING, enum: [...SCROLL_DIRECTIONS] };
|
|
78
|
+
function unimplementedOptionalActions(computer) {
|
|
79
|
+
return [
|
|
80
|
+
...OPTIONAL_MOUSE_ACTIONS.filter(
|
|
81
|
+
(name) => typeof computer[MOUSE_METHODS[name]] !== "function"
|
|
82
|
+
),
|
|
83
|
+
...OPTIONAL_KEY_ACTIONS.filter(
|
|
84
|
+
(name) => typeof computer[KEY_METHODS[name]] !== "function"
|
|
85
|
+
)
|
|
86
|
+
];
|
|
87
|
+
}
|
|
88
|
+
__name(unimplementedOptionalActions, "unimplementedOptionalActions");
|
|
89
|
+
function createComputerActions(options) {
|
|
90
|
+
return [
|
|
91
|
+
...createLegacyActions(options),
|
|
92
|
+
...createModernActions(options.computer),
|
|
93
|
+
...createOptionalActions(options.computer)
|
|
94
|
+
];
|
|
95
|
+
}
|
|
96
|
+
__name(createComputerActions, "createComputerActions");
|
|
97
|
+
function createLegacyActions({
|
|
98
|
+
computer,
|
|
99
|
+
allowPrivateNetworkAccess
|
|
100
|
+
}) {
|
|
101
|
+
return [
|
|
102
|
+
{
|
|
103
|
+
name: "open_web_browser",
|
|
104
|
+
description: "Opens the web browser.",
|
|
105
|
+
parameters: objectSchema({}),
|
|
106
|
+
execute: /* @__PURE__ */ __name(() => computer.openWebBrowser(), "execute")
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
name: "click_at",
|
|
110
|
+
description: "Clicks at a specific x, y coordinate on the webpage.",
|
|
111
|
+
parameters: objectSchema({ x: INTEGER, y: INTEGER }),
|
|
112
|
+
execute: /* @__PURE__ */ __name((args) => computer.clickAt(readNumber(args, "x"), readNumber(args, "y")), "execute")
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
name: "hover_at",
|
|
116
|
+
description: "Hovers at a specific x, y coordinate on the webpage. May be used to explore sub-menus that appear on hover.",
|
|
117
|
+
parameters: objectSchema({ x: INTEGER, y: INTEGER }),
|
|
118
|
+
execute: /* @__PURE__ */ __name((args) => computer.hoverAt(readNumber(args, "x"), readNumber(args, "y")), "execute")
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
name: "type_text_at",
|
|
122
|
+
description: "Types text at a specific x, y coordinate. Presses ENTER after typing unless press_enter is false, and clears existing content before typing unless clear_before_typing is false.",
|
|
123
|
+
parameters: objectSchema(
|
|
124
|
+
{
|
|
125
|
+
x: INTEGER,
|
|
126
|
+
y: INTEGER,
|
|
127
|
+
text: STRING,
|
|
128
|
+
press_enter: BOOLEAN,
|
|
129
|
+
clear_before_typing: BOOLEAN
|
|
130
|
+
},
|
|
131
|
+
["x", "y", "text"]
|
|
132
|
+
),
|
|
133
|
+
execute: /* @__PURE__ */ __name((args) => computer.typeTextAt(
|
|
134
|
+
readNumber(args, "x"),
|
|
135
|
+
readNumber(args, "y"),
|
|
136
|
+
readString(args, "text"),
|
|
137
|
+
readBoolean(args, "press_enter", true),
|
|
138
|
+
readBoolean(args, "clear_before_typing", true)
|
|
139
|
+
), "execute")
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
name: "scroll_document",
|
|
143
|
+
description: 'Scrolls the entire webpage "up", "down", "left" or "right" based on direction.',
|
|
144
|
+
parameters: objectSchema({ direction: DIRECTION }),
|
|
145
|
+
execute: /* @__PURE__ */ __name((args) => computer.scrollDocument(readDirection(args)), "execute")
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
name: "scroll_at",
|
|
149
|
+
description: "Scrolls up, down, right, or left at an x, y coordinate by magnitude.",
|
|
150
|
+
parameters: objectSchema({
|
|
151
|
+
x: INTEGER,
|
|
152
|
+
y: INTEGER,
|
|
153
|
+
direction: DIRECTION,
|
|
154
|
+
magnitude: INTEGER
|
|
155
|
+
}),
|
|
156
|
+
execute: /* @__PURE__ */ __name((args) => computer.scrollAt(
|
|
157
|
+
readNumber(args, "x"),
|
|
158
|
+
readNumber(args, "y"),
|
|
159
|
+
readDirection(args),
|
|
160
|
+
readNumber(args, "magnitude")
|
|
161
|
+
), "execute")
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
name: "wait",
|
|
165
|
+
description: "Waits for the specified number of seconds.",
|
|
166
|
+
parameters: objectSchema({ seconds: NUMBER }, []),
|
|
167
|
+
execute: /* @__PURE__ */ __name((args) => computer.wait(Math.max(0, Math.ceil(readNumber(args, "seconds", 1)))), "execute")
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
name: "go_back",
|
|
171
|
+
description: "Navigates back to the previous webpage in the browser history.",
|
|
172
|
+
parameters: objectSchema({}),
|
|
173
|
+
execute: /* @__PURE__ */ __name(() => computer.goBack(), "execute")
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
name: "go_forward",
|
|
177
|
+
description: "Navigates forward to the next webpage in the browser history.",
|
|
178
|
+
parameters: objectSchema({}),
|
|
179
|
+
execute: /* @__PURE__ */ __name(() => computer.goForward(), "execute")
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
name: "search",
|
|
183
|
+
description: "Directly jumps to a search engine home page. Used when you need to start with a search.",
|
|
184
|
+
parameters: objectSchema({}),
|
|
185
|
+
execute: /* @__PURE__ */ __name(() => computer.search(), "execute")
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
name: "navigate",
|
|
189
|
+
description: "Navigates directly to a specified URL.",
|
|
190
|
+
parameters: objectSchema({ url: STRING }),
|
|
191
|
+
execute: /* @__PURE__ */ __name((args) => navigate({ computer, url: args["url"], allowPrivateNetworkAccess }), "execute")
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
name: "key_combination",
|
|
195
|
+
description: 'Presses keyboard keys and combinations, such as "control+c" or "enter".',
|
|
196
|
+
parameters: objectSchema({ keys: STRING_ARRAY }),
|
|
197
|
+
execute: /* @__PURE__ */ __name((args) => computer.keyCombination(readStringArray(args, "keys")), "execute")
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
name: "drag_and_drop",
|
|
201
|
+
description: "Drags an element from an x, y coordinate and drops it at a destination_x, destination_y coordinate.",
|
|
202
|
+
parameters: objectSchema({
|
|
203
|
+
x: INTEGER,
|
|
204
|
+
y: INTEGER,
|
|
205
|
+
destination_x: INTEGER,
|
|
206
|
+
destination_y: INTEGER
|
|
207
|
+
}),
|
|
208
|
+
execute: /* @__PURE__ */ __name((args) => computer.dragAndDrop(
|
|
209
|
+
readNumber(args, "x", args["start_x"]),
|
|
210
|
+
readNumber(args, "y", args["start_y"]),
|
|
211
|
+
readNumber(args, "destination_x", args["end_x"]),
|
|
212
|
+
readNumber(args, "destination_y", args["end_y"])
|
|
213
|
+
), "execute")
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
name: "current_state",
|
|
217
|
+
description: "Returns the current state of the current webpage.",
|
|
218
|
+
parameters: objectSchema({}),
|
|
219
|
+
execute: /* @__PURE__ */ __name(() => computer.currentState(), "execute")
|
|
220
|
+
}
|
|
221
|
+
];
|
|
222
|
+
}
|
|
223
|
+
__name(createLegacyActions, "createLegacyActions");
|
|
224
|
+
function createModernActions(computer) {
|
|
225
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
226
|
+
const click = (_a = computer.click) == null ? void 0 : _a.bind(computer);
|
|
227
|
+
const move = (_b = computer.move) == null ? void 0 : _b.bind(computer);
|
|
228
|
+
const type = (_c = computer.type) == null ? void 0 : _c.bind(computer);
|
|
229
|
+
const scroll = (_d = computer.scroll) == null ? void 0 : _d.bind(computer);
|
|
230
|
+
const takeScreenshot = (_e = computer.takeScreenshot) == null ? void 0 : _e.bind(computer);
|
|
231
|
+
const pressKey = (_f = computer.pressKey) == null ? void 0 : _f.bind(computer);
|
|
232
|
+
const hotkey = (_g = computer.hotkey) == null ? void 0 : _g.bind(computer);
|
|
233
|
+
return [
|
|
234
|
+
{
|
|
235
|
+
name: "click",
|
|
236
|
+
description: "Clicks at the specified (x, y) coordinates.",
|
|
237
|
+
parameters: click ? objectSchema({ x: INTEGER, y: INTEGER, button: STRING }, ["x", "y"]) : objectSchema({ x: INTEGER, y: INTEGER }),
|
|
238
|
+
execute: /* @__PURE__ */ __name((args) => {
|
|
239
|
+
const x = readNumber(args, "x");
|
|
240
|
+
const y = readNumber(args, "y");
|
|
241
|
+
return click ? click(x, y, readOptional(args, "button", readString)) : computer.clickAt(x, y);
|
|
242
|
+
}, "execute")
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
name: "move",
|
|
246
|
+
description: "Moves the cursor to the specified (x, y) coordinates.",
|
|
247
|
+
parameters: objectSchema({ x: INTEGER, y: INTEGER }),
|
|
248
|
+
execute: /* @__PURE__ */ __name((args) => (move != null ? move : computer.hoverAt.bind(computer))(
|
|
249
|
+
readNumber(args, "x"),
|
|
250
|
+
readNumber(args, "y")
|
|
251
|
+
), "execute")
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
name: "type",
|
|
255
|
+
description: "Types text into the currently focused element.",
|
|
256
|
+
parameters: objectSchema({ text: STRING, press_enter: BOOLEAN }, ["text"]),
|
|
257
|
+
execute: /* @__PURE__ */ __name((args) => {
|
|
258
|
+
const text = readString(args, "text");
|
|
259
|
+
return type ? type(
|
|
260
|
+
text,
|
|
261
|
+
readOptional(
|
|
262
|
+
args,
|
|
263
|
+
"press_enter",
|
|
264
|
+
(a, n) => readBoolean(a, n, false)
|
|
265
|
+
)
|
|
266
|
+
) : computer.typeText(text, readBoolean(args, "press_enter", false));
|
|
267
|
+
}, "execute")
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
name: "scroll",
|
|
271
|
+
description: "Scrolls the webpage at the specified coordinates.",
|
|
272
|
+
parameters: objectSchema(
|
|
273
|
+
{
|
|
274
|
+
x: INTEGER,
|
|
275
|
+
y: INTEGER,
|
|
276
|
+
direction: DIRECTION,
|
|
277
|
+
magnitude_in_pixels: INTEGER
|
|
278
|
+
},
|
|
279
|
+
scroll ? ["x", "y", "direction"] : ["x", "y"]
|
|
280
|
+
),
|
|
281
|
+
execute: /* @__PURE__ */ __name((args) => {
|
|
282
|
+
const x = readNumber(args, "x");
|
|
283
|
+
const y = readNumber(args, "y");
|
|
284
|
+
return scroll ? scroll(
|
|
285
|
+
x,
|
|
286
|
+
y,
|
|
287
|
+
readDirection(args),
|
|
288
|
+
readOptional(args, "magnitude_in_pixels", readNumber)
|
|
289
|
+
) : computer.scrollAt(
|
|
290
|
+
x,
|
|
291
|
+
y,
|
|
292
|
+
readDirection(args, "down"),
|
|
293
|
+
readNumber(args, "magnitude_in_pixels", 300)
|
|
294
|
+
);
|
|
295
|
+
}, "execute")
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
name: "take_screenshot",
|
|
299
|
+
description: "Takes a screenshot of the current webpage state.",
|
|
300
|
+
parameters: objectSchema({}),
|
|
301
|
+
execute: /* @__PURE__ */ __name(() => takeScreenshot ? takeScreenshot() : computer.currentState(), "execute")
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
name: "press_key",
|
|
305
|
+
description: "Presses a single keyboard key.",
|
|
306
|
+
parameters: objectSchema({ key: STRING }),
|
|
307
|
+
execute: /* @__PURE__ */ __name((args) => {
|
|
308
|
+
const key = readString(args, "key");
|
|
309
|
+
return pressKey ? pressKey(key) : computer.keyCombination([key]);
|
|
310
|
+
}, "execute")
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
name: "hotkey",
|
|
314
|
+
description: "Presses a combination of keyboard keys simultaneously.",
|
|
315
|
+
parameters: objectSchema({ keys: STRING_ARRAY }),
|
|
316
|
+
execute: /* @__PURE__ */ __name((args) => (hotkey != null ? hotkey : computer.keyCombination.bind(computer))(
|
|
317
|
+
readStringArray(args, "keys")
|
|
318
|
+
), "execute")
|
|
319
|
+
}
|
|
320
|
+
];
|
|
321
|
+
}
|
|
322
|
+
__name(createModernActions, "createModernActions");
|
|
323
|
+
function createOptionalActions(computer) {
|
|
324
|
+
var _a, _b;
|
|
325
|
+
const actions = [];
|
|
326
|
+
for (const name of OPTIONAL_MOUSE_ACTIONS) {
|
|
327
|
+
const method = (_a = computer[MOUSE_METHODS[name]]) == null ? void 0 : _a.bind(computer);
|
|
328
|
+
if (!method) {
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
actions.push({
|
|
332
|
+
name,
|
|
333
|
+
description: `Performs ${name.replace("_", " ")} at the specified (x, y) coordinates.`,
|
|
334
|
+
parameters: objectSchema({ x: INTEGER, y: INTEGER, button: STRING }, [
|
|
335
|
+
"x",
|
|
336
|
+
"y"
|
|
337
|
+
]),
|
|
338
|
+
execute: /* @__PURE__ */ __name((args) => {
|
|
339
|
+
const button = args["button"];
|
|
340
|
+
return button === void 0 ? method(readNumber(args, "x"), readNumber(args, "y")) : method(
|
|
341
|
+
readNumber(args, "x"),
|
|
342
|
+
readNumber(args, "y"),
|
|
343
|
+
readString(args, "button")
|
|
344
|
+
);
|
|
345
|
+
}, "execute")
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
for (const name of OPTIONAL_KEY_ACTIONS) {
|
|
349
|
+
const method = (_b = computer[KEY_METHODS[name]]) == null ? void 0 : _b.bind(computer);
|
|
350
|
+
if (!method) {
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
actions.push({
|
|
354
|
+
name,
|
|
355
|
+
description: name === "key_down" ? "Presses a keyboard key down and holds it." : "Releases a keyboard key.",
|
|
356
|
+
parameters: objectSchema({ key: STRING }),
|
|
357
|
+
execute: /* @__PURE__ */ __name((args) => method(readString(args, "key")), "execute")
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
return actions;
|
|
361
|
+
}
|
|
362
|
+
__name(createOptionalActions, "createOptionalActions");
|
|
363
|
+
async function navigate({
|
|
364
|
+
computer,
|
|
365
|
+
url,
|
|
366
|
+
allowPrivateNetworkAccess
|
|
367
|
+
}) {
|
|
368
|
+
if (typeof url === "string" && await isSafeNavigationUrl(url, allowPrivateNetworkAccess)) {
|
|
369
|
+
return computer.navigate(url);
|
|
370
|
+
}
|
|
371
|
+
logger.warn("Refusing navigate(): url failed safety validation.");
|
|
372
|
+
const state = await computer.currentState();
|
|
373
|
+
return { error: URL_REFUSED_ERROR, url: state.url };
|
|
374
|
+
}
|
|
375
|
+
__name(navigate, "navigate");
|
|
376
|
+
async function isSafeNavigationUrl(url, allowPrivateNetworkAccess) {
|
|
377
|
+
var _a;
|
|
378
|
+
if ((_a = URL_AUTHORITY.exec(url)) == null ? void 0 : _a[1].includes("\\")) {
|
|
379
|
+
return false;
|
|
380
|
+
}
|
|
381
|
+
let parsed;
|
|
382
|
+
try {
|
|
383
|
+
parsed = new URL(url);
|
|
384
|
+
} catch {
|
|
385
|
+
return false;
|
|
386
|
+
}
|
|
387
|
+
const hostname = normalizeHost(parsed.hostname);
|
|
388
|
+
if (!isHttpUrl(parsed) || !hostname) {
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
if (allowPrivateNetworkAccess) {
|
|
392
|
+
return true;
|
|
393
|
+
}
|
|
394
|
+
if (isBlockedHostname(hostname)) {
|
|
395
|
+
return false;
|
|
396
|
+
}
|
|
397
|
+
if (isBrowser() && !isIpAddress(hostname)) {
|
|
398
|
+
return true;
|
|
399
|
+
}
|
|
400
|
+
try {
|
|
401
|
+
const addresses = await resolveHostAddresses(hostname);
|
|
402
|
+
return !addresses.some(isNonGlobalAddress);
|
|
403
|
+
} catch {
|
|
404
|
+
return false;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
__name(isSafeNavigationUrl, "isSafeNavigationUrl");
|
|
408
|
+
function isBlockedHostname(hostname) {
|
|
409
|
+
const normalized = hostname.replace(/\.+$/, "").toLowerCase();
|
|
410
|
+
return BLOCKED_HOSTNAMES.has(normalized) || BLOCKED_HOSTNAME_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
|
|
411
|
+
}
|
|
412
|
+
__name(isBlockedHostname, "isBlockedHostname");
|
|
413
|
+
function objectSchema(properties, required = Object.keys(properties)) {
|
|
414
|
+
return { type: Type.OBJECT, properties, required };
|
|
415
|
+
}
|
|
416
|
+
__name(objectSchema, "objectSchema");
|
|
417
|
+
function readNumber(args, name, fallback) {
|
|
418
|
+
var _a;
|
|
419
|
+
const value = (_a = args[name]) != null ? _a : fallback;
|
|
420
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
421
|
+
throw new Error(argumentError(name, "a number", value));
|
|
422
|
+
}
|
|
423
|
+
return value;
|
|
424
|
+
}
|
|
425
|
+
__name(readNumber, "readNumber");
|
|
426
|
+
function readString(args, name) {
|
|
427
|
+
const value = args[name];
|
|
428
|
+
if (typeof value !== "string") {
|
|
429
|
+
throw new Error(argumentError(name, "a string", value));
|
|
430
|
+
}
|
|
431
|
+
return value;
|
|
432
|
+
}
|
|
433
|
+
__name(readString, "readString");
|
|
434
|
+
function readBoolean(args, name, fallback) {
|
|
435
|
+
var _a;
|
|
436
|
+
const value = (_a = args[name]) != null ? _a : fallback;
|
|
437
|
+
if (typeof value !== "boolean") {
|
|
438
|
+
throw new Error(argumentError(name, "a boolean", value));
|
|
439
|
+
}
|
|
440
|
+
return value;
|
|
441
|
+
}
|
|
442
|
+
__name(readBoolean, "readBoolean");
|
|
443
|
+
function readOptional(args, name, read) {
|
|
444
|
+
return args[name] === void 0 ? void 0 : read(args, name);
|
|
445
|
+
}
|
|
446
|
+
__name(readOptional, "readOptional");
|
|
447
|
+
function readStringArray(args, name) {
|
|
448
|
+
const value = args[name];
|
|
449
|
+
if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) {
|
|
450
|
+
throw new Error(argumentError(name, "a list of strings", value));
|
|
451
|
+
}
|
|
452
|
+
return value;
|
|
453
|
+
}
|
|
454
|
+
__name(readStringArray, "readStringArray");
|
|
455
|
+
function readDirection(args, fallback) {
|
|
456
|
+
var _a;
|
|
457
|
+
const value = (_a = args["direction"]) != null ? _a : fallback;
|
|
458
|
+
const direction = SCROLL_DIRECTIONS.find((d) => d === value);
|
|
459
|
+
if (!direction) {
|
|
460
|
+
throw new Error(
|
|
461
|
+
argumentError("direction", "one of up, down, left, right", value)
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
return direction;
|
|
465
|
+
}
|
|
466
|
+
__name(readDirection, "readDirection");
|
|
467
|
+
function argumentError(name, expected, value) {
|
|
468
|
+
return value === void 0 ? `Missing required argument '${name}'.` : `Argument '${name}' must be ${expected}, got ${typeof value}.`;
|
|
469
|
+
}
|
|
470
|
+
__name(argumentError, "argumentError");
|
|
471
|
+
export {
|
|
472
|
+
LEGACY_ACTION_NAMES,
|
|
473
|
+
OPTIONAL_KEY_ACTIONS,
|
|
474
|
+
OPTIONAL_MOUSE_ACTIONS,
|
|
475
|
+
createComputerActions,
|
|
476
|
+
unimplementedOptionalActions
|
|
477
|
+
};
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import {createRequire as topLevelCreateRequire} from 'module';
|
|
2
|
+
const require = topLevelCreateRequire(import.meta.url);
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
6
|
+
var __decorateClass = (decorators, target, key, kind) => {
|
|
7
|
+
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
8
|
+
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
9
|
+
if (decorator = decorators[i])
|
|
10
|
+
result = (kind ? decorator(target, key, result) : decorator(result)) || result;
|
|
11
|
+
if (kind && result) __defProp(target, key, result);
|
|
12
|
+
return result;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* @license
|
|
16
|
+
* Copyright 2026 Google LLC
|
|
17
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
18
|
+
*/
|
|
19
|
+
var _a, _b;
|
|
20
|
+
import { base64Encode } from "../../utils/env_aware_utils.js";
|
|
21
|
+
import { experimental } from "../../utils/experimental.js";
|
|
22
|
+
import { logger } from "../../utils/logger.js";
|
|
23
|
+
import { FunctionTool } from "../function_tool.js";
|
|
24
|
+
import { checkComputerUseEnabled } from "./base_computer.js";
|
|
25
|
+
const DEFAULT_VIRTUAL_SCREEN_SIZE = [1e3, 1e3];
|
|
26
|
+
const X_COORDINATE_FIELDS = ["x", "destination_x", "start_x", "end_x"];
|
|
27
|
+
const Y_COORDINATE_FIELDS = ["y", "destination_y", "start_y", "end_y"];
|
|
28
|
+
const INTEGER_STRING = /^-?\d+$/;
|
|
29
|
+
const REQUIRE_CONFIRMATION = "require_confirmation";
|
|
30
|
+
const COMPUTER_USE_TOOL_SIGNATURE_SYMBOL = /* @__PURE__ */ Symbol.for(
|
|
31
|
+
"google.adk.computerUseTool"
|
|
32
|
+
);
|
|
33
|
+
function isComputerUseTool(obj) {
|
|
34
|
+
return typeof obj === "object" && obj !== null && COMPUTER_USE_TOOL_SIGNATURE_SYMBOL in obj && obj[COMPUTER_USE_TOOL_SIGNATURE_SYMBOL] === true;
|
|
35
|
+
}
|
|
36
|
+
__name(isComputerUseTool, "isComputerUseTool");
|
|
37
|
+
let ComputerUseTool = class extends (_b = FunctionTool, _a = COMPUTER_USE_TOOL_SIGNATURE_SYMBOL, _b) {
|
|
38
|
+
/**
|
|
39
|
+
* @param options The options for the tool.
|
|
40
|
+
* @throws {Error} If `screenSize` or `virtualScreenSize` is not a pair of
|
|
41
|
+
* positive numbers.
|
|
42
|
+
*/
|
|
43
|
+
constructor(options) {
|
|
44
|
+
var _a2;
|
|
45
|
+
checkComputerUseEnabled();
|
|
46
|
+
const virtualScreenSize = (_a2 = options.virtualScreenSize) != null ? _a2 : DEFAULT_VIRTUAL_SCREEN_SIZE;
|
|
47
|
+
checkScreenSize("screenSize", options.screenSize);
|
|
48
|
+
checkScreenSize("virtualScreenSize", virtualScreenSize);
|
|
49
|
+
const func = options.execute;
|
|
50
|
+
super({
|
|
51
|
+
name: options.name,
|
|
52
|
+
description: options.description,
|
|
53
|
+
parameters: options.parameters,
|
|
54
|
+
execute: /* @__PURE__ */ __name((input, toolContext) => func(isRecord(input) ? input : {}, toolContext), "execute")
|
|
55
|
+
});
|
|
56
|
+
/** A unique symbol to identify ADK computer use tool class. */
|
|
57
|
+
this[_a] = true;
|
|
58
|
+
this.func = func;
|
|
59
|
+
this.screenSize = options.screenSize;
|
|
60
|
+
this.virtualScreenSize = virtualScreenSize;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Runs the computer control function with scaled coordinates.
|
|
64
|
+
*
|
|
65
|
+
* @param request The tool request containing arguments and tool context.
|
|
66
|
+
* @returns The function's result, with a {@link ComputerState} converted to
|
|
67
|
+
* a response carrying the base64-encoded screenshot.
|
|
68
|
+
* @throws {Error} If a coordinate is not numeric, or the function returns a
|
|
69
|
+
* state with no screenshot.
|
|
70
|
+
*/
|
|
71
|
+
async runAsync({
|
|
72
|
+
args,
|
|
73
|
+
toolContext
|
|
74
|
+
}) {
|
|
75
|
+
const confirmation = toolContext.toolConfirmation;
|
|
76
|
+
if (!confirmation) {
|
|
77
|
+
const safetyDecision = getSafetyDecision(args);
|
|
78
|
+
if ((safetyDecision == null ? void 0 : safetyDecision.decision) === REQUIRE_CONFIRMATION) {
|
|
79
|
+
toolContext.requestConfirmation({
|
|
80
|
+
hint: typeof safetyDecision.explanation === "string" && safetyDecision.explanation ? safetyDecision.explanation : "This computer use action requires safety confirmation."
|
|
81
|
+
});
|
|
82
|
+
toolContext.actions.skipSummarization = true;
|
|
83
|
+
return {
|
|
84
|
+
error: "This tool call requires confirmation, please approve or reject."
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
} else if (!confirmation.confirmed) {
|
|
88
|
+
return { error: "This tool call is rejected." };
|
|
89
|
+
}
|
|
90
|
+
const scaledArgs = this.scaleCoordinates(args);
|
|
91
|
+
const result = await super.runAsync({ args: scaledArgs, toolContext });
|
|
92
|
+
let response = result;
|
|
93
|
+
if (isComputerState(result)) {
|
|
94
|
+
if (!result.screenshot) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
"A computer action returned a state with no screenshot"
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
response = {
|
|
100
|
+
image: { mimetype: "image/png", data: base64Encode(result.screenshot) },
|
|
101
|
+
url: result.url
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
if (confirmation == null ? void 0 : confirmation.confirmed) {
|
|
105
|
+
const acknowledged = isRecord(response) ? response : { result: response };
|
|
106
|
+
acknowledged["safety_acknowledgement"] = "true";
|
|
107
|
+
return acknowledged;
|
|
108
|
+
}
|
|
109
|
+
return response;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Whether this call needs user confirmation: when the model marked it with
|
|
113
|
+
* a `safety_decision` of `require_confirmation`, or when the base tool says
|
|
114
|
+
* so.
|
|
115
|
+
*
|
|
116
|
+
* @param args The arguments the tool would run with.
|
|
117
|
+
* @param toolContext The context of the call, when there is one.
|
|
118
|
+
* @returns Whether the call requires confirmation.
|
|
119
|
+
*/
|
|
120
|
+
async checkRequireConfirmation(args, toolContext) {
|
|
121
|
+
var _a2;
|
|
122
|
+
if (((_a2 = getSafetyDecision(args)) == null ? void 0 : _a2.decision) === REQUIRE_CONFIRMATION) {
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
return super.checkRequireConfirmation(args, toolContext);
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Adds nothing to the request: {@link ComputerUseToolset} adds the single
|
|
129
|
+
* computer use configuration and registers its tools by name.
|
|
130
|
+
*/
|
|
131
|
+
async processLlmRequest(_request) {
|
|
132
|
+
}
|
|
133
|
+
/** Returns a copy of `args` with every coordinate scaled to the screen. */
|
|
134
|
+
scaleCoordinates(args) {
|
|
135
|
+
const scaled = { ...args };
|
|
136
|
+
delete scaled["safety_decision"];
|
|
137
|
+
for (const field of X_COORDINATE_FIELDS) {
|
|
138
|
+
if (field in scaled) {
|
|
139
|
+
scaled[field] = scaleCoordinate({
|
|
140
|
+
field,
|
|
141
|
+
value: scaled[field],
|
|
142
|
+
virtual: this.virtualScreenSize[0],
|
|
143
|
+
actual: this.screenSize[0]
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const field of Y_COORDINATE_FIELDS) {
|
|
148
|
+
if (field in scaled) {
|
|
149
|
+
scaled[field] = scaleCoordinate({
|
|
150
|
+
field,
|
|
151
|
+
value: scaled[field],
|
|
152
|
+
virtual: this.virtualScreenSize[1],
|
|
153
|
+
actual: this.screenSize[1]
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return scaled;
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
__name(ComputerUseTool, "ComputerUseTool");
|
|
161
|
+
ComputerUseTool = __decorateClass([
|
|
162
|
+
experimental
|
|
163
|
+
], ComputerUseTool);
|
|
164
|
+
function scaleCoordinate({
|
|
165
|
+
field,
|
|
166
|
+
value,
|
|
167
|
+
virtual,
|
|
168
|
+
actual
|
|
169
|
+
}) {
|
|
170
|
+
const numeric = typeof value === "string" && INTEGER_STRING.test(value) ? Number(value) : value;
|
|
171
|
+
if (typeof numeric !== "number" || !Number.isFinite(numeric)) {
|
|
172
|
+
throw new Error(
|
|
173
|
+
`${field} coordinate must be numeric, got ${typeof value} ${JSON.stringify(value)}`
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
const scaled = Math.max(
|
|
177
|
+
0,
|
|
178
|
+
Math.min(Math.trunc(numeric / virtual * actual), actual - 1)
|
|
179
|
+
);
|
|
180
|
+
logger.debug(`Normalized ${field}: ${numeric} -> ${scaled}`);
|
|
181
|
+
return scaled;
|
|
182
|
+
}
|
|
183
|
+
__name(scaleCoordinate, "scaleCoordinate");
|
|
184
|
+
function checkScreenSize(name, size) {
|
|
185
|
+
if (!Array.isArray(size) || size.length !== 2) {
|
|
186
|
+
throw new Error(`${name} must be a pair of (width, height)`);
|
|
187
|
+
}
|
|
188
|
+
if (!size.every((dimension) => Number.isFinite(dimension) && dimension > 0)) {
|
|
189
|
+
throw new Error(`${name} dimensions must be positive`);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
__name(checkScreenSize, "checkScreenSize");
|
|
193
|
+
function getSafetyDecision(args) {
|
|
194
|
+
const decision = args["safety_decision"];
|
|
195
|
+
return isRecord(decision) ? decision : void 0;
|
|
196
|
+
}
|
|
197
|
+
__name(getSafetyDecision, "getSafetyDecision");
|
|
198
|
+
function isRecord(value) {
|
|
199
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
200
|
+
}
|
|
201
|
+
__name(isRecord, "isRecord");
|
|
202
|
+
function isComputerState(value) {
|
|
203
|
+
if (!isRecord(value)) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
const keys = Object.keys(value);
|
|
207
|
+
if (keys.length === 0 || keys.some((k) => k !== "screenshot" && k !== "url")) {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
const { screenshot, url } = value;
|
|
211
|
+
return (screenshot === void 0 || screenshot instanceof Uint8Array) && (url === void 0 || typeof url === "string");
|
|
212
|
+
}
|
|
213
|
+
__name(isComputerState, "isComputerState");
|
|
214
|
+
export {
|
|
215
|
+
ComputerUseTool,
|
|
216
|
+
isComputerUseTool
|
|
217
|
+
};
|