@google/adk 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/a2a/a2a_remote_agent.js +17 -2
- package/dist/cjs/a2a/a2a_remote_agent_utils.js +8 -4
- package/dist/cjs/a2a/agent_card.js +5 -81
- package/dist/cjs/a2a/agent_to_a2a.js +7 -0
- package/dist/cjs/a2a/event_converter_utils.js +10 -3
- package/dist/cjs/agents/agent_engine_client.js +282 -55
- package/dist/cjs/agents/base_agent.js +111 -68
- package/dist/cjs/agents/functions.js +62 -85
- package/dist/cjs/agents/instructions.js +7 -6
- package/dist/cjs/agents/invocation_context.js +2 -2
- package/dist/cjs/agents/llm_agent.js +105 -95
- package/dist/cjs/agents/processors/content_processor_utils.js +42 -15
- package/dist/cjs/agents/processors/content_request_processor.js +28 -11
- package/dist/cjs/agents/processors/context_compactor_request_processor.js +5 -1
- package/dist/cjs/agents/processors/instructions_llm_request_processor.js +1 -1
- package/dist/cjs/agents/processors/nl_planning_processor.js +2 -1
- package/dist/cjs/artifacts/file_artifact_service.js +1 -1
- package/dist/cjs/cli/cli_eval.js +884 -17
- package/dist/cjs/cli/utils/cleanup.js +84 -0
- package/dist/cjs/cli/utils/envs.js +33 -2
- package/dist/cjs/cli/utils/evals.js +75 -8
- package/dist/cjs/cli/utils/logs.js +128 -9
- package/dist/cjs/cli/utils/path_validation.js +83 -0
- package/dist/cjs/code_executors/container_code_executor.js +158 -36
- package/dist/cjs/common.js +50 -12
- package/dist/cjs/context/compaction_utils.js +19 -0
- package/dist/cjs/events/event.js +7 -3
- package/dist/cjs/features/feature_registry.js +5 -0
- package/dist/cjs/index.js +29 -14
- package/dist/cjs/models/google_llm.js +33 -2
- package/dist/cjs/models/interactions_utils.js +3 -0
- package/dist/cjs/plugins/base_plugin.js +45 -21
- package/dist/cjs/plugins/plugin_manager.js +27 -0
- package/dist/cjs/runner/runner.js +306 -189
- package/dist/cjs/sessions/base_session_service.js +1 -4
- package/dist/cjs/sessions/database_session_service.js +191 -19
- package/dist/cjs/sessions/db/operations.js +70 -1
- package/dist/cjs/sessions/db/schema.js +4 -0
- package/dist/cjs/sessions/in_memory_session_service.js +2 -2
- package/dist/cjs/sessions/vertex_ai_session_service.js +11 -3
- package/dist/cjs/skills/loader.js +29 -28
- package/dist/cjs/telemetry/api/instrumentation.js +632 -0
- package/dist/cjs/telemetry/api/internal/common/accumulators.js +169 -0
- package/dist/cjs/telemetry/api/internal/common/attributes.js +131 -0
- package/dist/cjs/telemetry/api/internal/common/config.js +159 -0
- package/dist/cjs/telemetry/api/internal/common/error.js +98 -0
- package/dist/cjs/telemetry/api/internal/common/finish_reason.js +63 -0
- package/dist/cjs/telemetry/api/internal/common/genai_messages.js +217 -0
- package/dist/cjs/telemetry/api/internal/common/http_exchange.js +169 -0
- package/dist/cjs/telemetry/api/internal/common/provider_name.js +87 -0
- package/dist/cjs/telemetry/api/internal/common/redaction.js +148 -0
- package/dist/cjs/telemetry/api/internal/common/serialization.js +58 -0
- package/dist/cjs/telemetry/api/internal/common/token_usage.js +118 -0
- package/dist/cjs/telemetry/api/internal/logs/inference.js +50 -0
- package/dist/cjs/telemetry/api/internal/logs/logger.js +46 -0
- package/dist/cjs/telemetry/api/internal/logs/mcp.js +141 -0
- package/dist/cjs/telemetry/api/internal/metrics/agent.js +128 -0
- package/dist/cjs/telemetry/api/internal/metrics/inference_start_hook.js +56 -0
- package/dist/cjs/telemetry/api/internal/metrics/meters.js +189 -0
- package/dist/cjs/telemetry/api/internal/metrics/tool.js +147 -0
- package/dist/cjs/telemetry/api/internal/metrics/workflow.js +132 -0
- package/dist/cjs/telemetry/api/internal/trace/agent.js +52 -0
- package/dist/cjs/telemetry/api/internal/trace/compaction.js +78 -0
- package/dist/cjs/telemetry/api/internal/trace/inference.js +132 -0
- package/dist/cjs/telemetry/api/internal/trace/live.js +52 -0
- package/dist/cjs/telemetry/api/internal/trace/node.js +73 -0
- package/dist/cjs/telemetry/api/internal/trace/propagation.js +67 -0
- package/dist/cjs/telemetry/api/internal/trace/tool.js +122 -0
- package/dist/cjs/telemetry/api/internal/trace/tracer.js +77 -0
- package/dist/cjs/telemetry/api/internal/trace/workflow.js +77 -0
- package/dist/cjs/telemetry/export/agent_engine.js +90 -0
- package/dist/cjs/telemetry/export/agent_runtime.js +73 -0
- package/dist/cjs/telemetry/export/google_cloud.js +763 -0
- package/dist/cjs/telemetry/export/request_driven_metric_reader.js +303 -0
- package/dist/cjs/telemetry/{setup.js → export/setup.js} +72 -5
- package/dist/cjs/tools/agent_tool.js +9 -1
- package/dist/cjs/tools/application_integration_tool/application_integration_toolset.js +22 -34
- package/dist/cjs/tools/application_integration_tool/clients/connections_client.js +60 -20
- package/dist/cjs/tools/application_integration_tool/clients/integration_client.js +26 -10
- package/dist/cjs/tools/application_integration_tool/integration_connector_tool.js +14 -23
- package/dist/cjs/tools/base_toolset.js +1 -1
- package/dist/cjs/tools/computer_use/base_computer.js +125 -0
- package/dist/cjs/tools/computer_use/computer_use_actions.js +502 -0
- package/dist/cjs/tools/computer_use/computer_use_tool.js +244 -0
- package/dist/cjs/tools/computer_use/computer_use_toolset.js +360 -0
- package/dist/cjs/tools/function_tool.js +15 -5
- package/dist/cjs/tools/google_api_tool/google_api_toolset.js +3 -2
- package/dist/cjs/tools/google_api_tool/googleapi_to_openapi_converter.js +3 -1
- package/dist/cjs/tools/load_artifacts_tool.js +15 -5
- package/dist/cjs/tools/load_memory_tool.js +2 -5
- package/dist/cjs/tools/mcp/mcp_session_manager.js +23 -3
- package/dist/cjs/tools/mcp/mcp_tool.js +7 -3
- package/dist/cjs/tools/mcp/mcp_toolset.js +18 -6
- package/dist/cjs/tools/openapi_tool/auth/auth_helpers.js +30 -29
- package/dist/cjs/tools/openapi_tool/auth/credential_exchangers/oauth2_exchanger.js +2 -2
- package/dist/cjs/tools/openapi_tool/rest_api_tool.js +45 -14
- package/dist/cjs/tools/skill/load_skill_resource_tool.js +7 -0
- package/dist/cjs/tools/skill/load_skill_tool.js +3 -0
- package/dist/cjs/tools/skill/run_skill_script_tool.js +12 -2
- package/dist/cjs/tools/toolbox_toolset.js +47 -28
- package/dist/cjs/utils/async_queue.js +37 -3
- package/dist/cjs/utils/client_labels.js +9 -11
- package/dist/cjs/utils/env_aware_utils.js +13 -7
- package/dist/cjs/utils/error_utils.js +40 -2
- package/dist/cjs/utils/failover_utils.js +10 -9
- package/dist/cjs/utils/gemini_schema_util.js +125 -12
- package/dist/cjs/utils/id_provider.js +63 -0
- package/dist/cjs/utils/model_name.js +15 -0
- package/dist/cjs/utils/simple_zod_to_json.js +13 -31
- package/dist/cjs/utils/ssrf_guard.js +46 -16
- package/dist/cjs/version.js +1 -1
- package/dist/cjs/workflow/index.js +2 -0
- package/dist/cjs/workflow/node_runner.js +24 -80
- package/dist/cjs/workflow/nodes/parallel_worker.js +18 -7
- package/dist/cjs/workflow/run_llm_agent_as_node.js +19 -14
- package/dist/cjs/workflow/utils/retry_utils.js +2 -2
- package/dist/cjs/workflow/workflow.js +12 -12
- package/dist/esm/a2a/a2a_remote_agent.js +23 -3
- package/dist/esm/a2a/a2a_remote_agent_utils.js +8 -4
- package/dist/esm/a2a/agent_card.js +5 -83
- package/dist/esm/a2a/agent_to_a2a.js +7 -0
- package/dist/esm/a2a/event_converter_utils.js +10 -3
- package/dist/esm/agents/agent_engine_client.js +293 -55
- package/dist/esm/agents/base_agent.js +111 -72
- package/dist/esm/agents/functions.js +67 -89
- package/dist/esm/agents/instructions.js +7 -6
- package/dist/esm/agents/invocation_context.js +2 -2
- package/dist/esm/agents/llm_agent.js +111 -100
- package/dist/esm/agents/processors/content_processor_utils.js +42 -15
- package/dist/esm/agents/processors/content_request_processor.js +28 -11
- package/dist/esm/agents/processors/context_compactor_request_processor.js +5 -1
- package/dist/esm/agents/processors/instructions_llm_request_processor.js +1 -1
- package/dist/esm/agents/processors/nl_planning_processor.js +2 -1
- package/dist/esm/artifacts/file_artifact_service.js +1 -1
- package/dist/esm/cli/cli_eval.js +880 -17
- package/dist/esm/cli/utils/cleanup.js +56 -0
- package/dist/esm/cli/utils/envs.js +33 -2
- package/dist/esm/cli/utils/evals.js +74 -8
- package/dist/esm/cli/utils/logs.js +128 -9
- package/dist/esm/cli/utils/path_validation.js +45 -0
- package/dist/esm/code_executors/container_code_executor.js +158 -36
- package/dist/esm/common.js +49 -6
- package/dist/esm/context/compaction_utils.js +19 -0
- package/dist/esm/events/event.js +7 -3
- package/dist/esm/features/feature_registry.js +5 -0
- package/dist/esm/index.js +27 -15
- package/dist/esm/models/google_llm.js +33 -2
- package/dist/esm/models/interactions_utils.js +3 -0
- package/dist/esm/plugins/base_plugin.js +45 -21
- package/dist/esm/plugins/plugin_manager.js +27 -0
- package/dist/esm/runner/runner.js +310 -192
- package/dist/esm/sessions/base_session_service.js +1 -4
- package/dist/esm/sessions/database_session_service.js +191 -19
- package/dist/esm/sessions/db/operations.js +70 -1
- package/dist/esm/sessions/db/schema.js +4 -0
- package/dist/esm/sessions/in_memory_session_service.js +2 -2
- package/dist/esm/sessions/vertex_ai_session_service.js +11 -3
- package/dist/esm/skills/loader.js +29 -28
- package/dist/esm/telemetry/api/instrumentation.js +655 -0
- package/dist/esm/telemetry/api/internal/common/accumulators.js +139 -0
- package/dist/esm/telemetry/api/internal/common/attributes.js +94 -0
- package/dist/esm/telemetry/api/internal/common/config.js +128 -0
- package/dist/esm/telemetry/api/internal/common/error.js +69 -0
- package/dist/esm/telemetry/api/internal/common/finish_reason.js +35 -0
- package/dist/esm/telemetry/api/internal/common/genai_messages.js +188 -0
- package/dist/esm/telemetry/api/internal/common/http_exchange.js +142 -0
- package/dist/esm/telemetry/api/internal/common/provider_name.js +59 -0
- package/dist/esm/telemetry/api/internal/common/redaction.js +121 -0
- package/dist/esm/telemetry/api/internal/common/serialization.js +30 -0
- package/dist/esm/telemetry/api/internal/common/token_usage.js +85 -0
- package/dist/esm/telemetry/api/internal/logs/inference.js +23 -0
- package/dist/esm/telemetry/api/internal/logs/logger.js +19 -0
- package/dist/esm/telemetry/api/internal/logs/mcp.js +113 -0
- package/dist/esm/telemetry/api/internal/metrics/agent.js +107 -0
- package/dist/esm/telemetry/api/internal/metrics/inference_start_hook.js +29 -0
- package/dist/esm/telemetry/api/internal/metrics/meters.js +165 -0
- package/dist/esm/telemetry/api/internal/metrics/tool.js +119 -0
- package/dist/esm/telemetry/api/internal/metrics/workflow.js +112 -0
- package/dist/esm/telemetry/api/internal/trace/agent.js +25 -0
- package/dist/esm/telemetry/api/internal/trace/compaction.js +50 -0
- package/dist/esm/telemetry/api/internal/trace/inference.js +107 -0
- package/dist/esm/telemetry/api/internal/trace/live.js +28 -0
- package/dist/esm/telemetry/api/internal/trace/node.js +44 -0
- package/dist/esm/telemetry/api/internal/trace/propagation.js +39 -0
- package/dist/esm/telemetry/api/internal/trace/tool.js +96 -0
- package/dist/esm/telemetry/api/internal/trace/tracer.js +52 -0
- package/dist/esm/telemetry/api/internal/trace/workflow.js +54 -0
- package/dist/esm/telemetry/export/agent_engine.js +67 -0
- package/dist/esm/telemetry/export/agent_runtime.js +46 -0
- package/dist/esm/telemetry/export/google_cloud.js +733 -0
- package/dist/esm/telemetry/export/request_driven_metric_reader.js +276 -0
- package/dist/esm/telemetry/export/setup.js +160 -0
- package/dist/esm/tools/agent_tool.js +9 -1
- package/dist/esm/tools/application_integration_tool/application_integration_toolset.js +23 -35
- package/dist/esm/tools/application_integration_tool/clients/connections_client.js +64 -21
- package/dist/esm/tools/application_integration_tool/clients/integration_client.js +26 -10
- package/dist/esm/tools/application_integration_tool/integration_connector_tool.js +14 -23
- package/dist/esm/tools/base_toolset.js +1 -1
- package/dist/esm/tools/computer_use/base_computer.js +99 -0
- package/dist/esm/tools/computer_use/computer_use_actions.js +477 -0
- package/dist/esm/tools/computer_use/computer_use_tool.js +217 -0
- package/dist/esm/tools/computer_use/computer_use_toolset.js +344 -0
- package/dist/esm/tools/function_tool.js +15 -5
- package/dist/esm/tools/google_api_tool/google_api_toolset.js +3 -2
- package/dist/esm/tools/google_api_tool/googleapi_to_openapi_converter.js +3 -1
- package/dist/esm/tools/load_artifacts_tool.js +15 -5
- package/dist/esm/tools/load_memory_tool.js +2 -5
- package/dist/esm/tools/mcp/mcp_session_manager.js +23 -3
- package/dist/esm/tools/mcp/mcp_tool.js +7 -3
- package/dist/esm/tools/mcp/mcp_toolset.js +18 -6
- package/dist/esm/tools/openapi_tool/auth/auth_helpers.js +27 -26
- package/dist/esm/tools/openapi_tool/auth/credential_exchangers/oauth2_exchanger.js +2 -2
- package/dist/esm/tools/openapi_tool/rest_api_tool.js +45 -14
- package/dist/esm/tools/skill/load_skill_resource_tool.js +7 -0
- package/dist/esm/tools/skill/load_skill_tool.js +3 -0
- package/dist/esm/tools/skill/run_skill_script_tool.js +12 -2
- package/dist/esm/tools/toolbox_toolset.js +47 -28
- package/dist/esm/utils/async_queue.js +37 -3
- package/dist/esm/utils/client_labels.js +9 -11
- package/dist/esm/utils/env_aware_utils.js +12 -7
- package/dist/esm/utils/error_utils.js +37 -1
- package/dist/esm/utils/failover_utils.js +10 -9
- package/dist/esm/utils/gemini_schema_util.js +124 -11
- package/dist/esm/utils/id_provider.js +33 -0
- package/dist/esm/utils/model_name.js +14 -0
- package/dist/esm/utils/simple_zod_to_json.js +13 -31
- package/dist/esm/utils/ssrf_guard.js +55 -15
- package/dist/esm/version.js +1 -1
- package/dist/esm/workflow/index.js +2 -1
- package/dist/esm/workflow/node_runner.js +26 -80
- package/dist/esm/workflow/nodes/parallel_worker.js +16 -6
- package/dist/esm/workflow/run_llm_agent_as_node.js +19 -14
- package/dist/esm/workflow/utils/retry_utils.js +2 -2
- package/dist/esm/workflow/workflow.js +12 -12
- package/dist/types/a2a/a2a_remote_agent.d.ts +6 -0
- package/dist/types/a2a/a2a_remote_agent_utils.d.ts +3 -2
- package/dist/types/agents/agent_engine_client.d.ts +121 -3
- package/dist/types/agents/base_agent.d.ts +2 -0
- package/dist/types/agents/processors/content_processor_utils.d.ts +9 -9
- package/dist/types/agents/run_config.d.ts +14 -0
- package/dist/types/cli/cli_eval.d.ts +172 -9
- package/dist/types/cli/utils/cleanup.d.ts +18 -0
- package/dist/types/cli/utils/envs.d.ts +13 -3
- package/dist/types/cli/utils/evals.d.ts +64 -1
- package/dist/types/cli/utils/logs.d.ts +19 -2
- package/dist/types/cli/utils/path_validation.d.ts +30 -0
- package/dist/types/code_executors/container_code_executor.d.ts +49 -8
- package/dist/types/common.d.ts +16 -9
- package/dist/types/context/compaction_utils.d.ts +13 -1
- package/dist/types/events/event.d.ts +3 -0
- package/dist/types/features/feature_registry.d.ts +1 -0
- package/dist/types/index.d.ts +9 -6
- package/dist/types/plugins/base_plugin.d.ts +50 -21
- package/dist/types/plugins/plugin_manager.d.ts +13 -0
- package/dist/types/runner/runner.d.ts +11 -0
- package/dist/types/sessions/base_session_service.d.ts +0 -2
- package/dist/types/sessions/db/operations.d.ts +2 -1
- package/dist/types/sessions/db/schema.d.ts +5 -3
- package/dist/types/telemetry/api/instrumentation.d.ts +259 -0
- package/dist/types/telemetry/api/internal/common/accumulators.d.ts +91 -0
- package/dist/types/telemetry/api/internal/common/attributes.d.ts +90 -0
- package/dist/types/telemetry/api/internal/common/config.d.ts +61 -0
- package/dist/types/telemetry/api/internal/common/error.d.ts +29 -0
- package/dist/types/telemetry/api/internal/common/finish_reason.d.ts +18 -0
- package/dist/types/telemetry/api/internal/common/genai_messages.d.ts +26 -0
- package/dist/types/telemetry/api/internal/common/http_exchange.d.ts +50 -0
- package/dist/types/telemetry/api/internal/common/provider_name.d.ts +16 -0
- package/dist/types/telemetry/api/internal/common/redaction.d.ts +14 -0
- package/dist/types/telemetry/api/internal/common/serialization.d.ts +18 -0
- package/dist/types/telemetry/api/internal/common/token_usage.d.ts +47 -0
- package/dist/types/telemetry/api/internal/logs/inference.d.ts +15 -0
- package/dist/types/telemetry/api/internal/logs/logger.d.ts +13 -0
- package/dist/types/telemetry/api/internal/logs/mcp.d.ts +31 -0
- package/dist/types/telemetry/api/internal/metrics/agent.d.ts +28 -0
- package/dist/types/telemetry/api/internal/metrics/inference_start_hook.d.ts +12 -0
- package/dist/types/telemetry/api/internal/metrics/meters.d.ts +57 -0
- package/dist/types/telemetry/api/internal/metrics/tool.d.ts +44 -0
- package/dist/types/telemetry/api/internal/metrics/workflow.d.ts +32 -0
- package/dist/types/telemetry/api/internal/trace/agent.d.ts +14 -0
- package/dist/types/telemetry/api/internal/trace/compaction.d.ts +20 -0
- package/dist/types/telemetry/api/internal/trace/inference.d.ts +31 -0
- package/dist/types/telemetry/api/internal/trace/live.d.ts +21 -0
- package/dist/types/telemetry/api/internal/trace/node.d.ts +22 -0
- package/dist/types/telemetry/api/internal/trace/propagation.d.ts +22 -0
- package/dist/types/telemetry/api/internal/trace/tool.d.ts +57 -0
- package/dist/types/telemetry/api/internal/trace/tracer.d.ts +31 -0
- package/dist/types/telemetry/api/internal/trace/workflow.d.ts +21 -0
- package/dist/types/telemetry/export/agent_engine.d.ts +23 -0
- package/dist/types/telemetry/export/agent_runtime.d.ts +40 -0
- package/dist/types/telemetry/export/google_cloud.d.ts +189 -0
- package/dist/types/telemetry/export/request_driven_metric_reader.d.ts +144 -0
- package/dist/types/telemetry/{setup.d.ts → export/setup.d.ts} +3 -0
- package/dist/types/tools/agent_tool.d.ts +1 -1
- package/dist/types/tools/application_integration_tool/clients/connections_client.d.ts +13 -4
- package/dist/types/tools/base_toolset.d.ts +1 -1
- package/dist/types/tools/computer_use/base_computer.d.ts +281 -0
- package/dist/types/tools/computer_use/computer_use_actions.d.ts +53 -0
- package/dist/types/tools/computer_use/computer_use_tool.d.ts +108 -0
- package/dist/types/tools/computer_use/computer_use_toolset.d.ts +132 -0
- package/dist/types/tools/google_api_tool/google_api_toolset.d.ts +2 -0
- package/dist/types/tools/load_artifacts_tool.d.ts +5 -1
- package/dist/types/tools/load_memory_tool.d.ts +13 -7
- package/dist/types/tools/mcp/mcp_session_manager.d.ts +16 -0
- package/dist/types/tools/openapi_tool/auth/auth_helpers.d.ts +8 -8
- package/dist/types/tools/openapi_tool/openapi_spec_parser/operation_parser.d.ts +7 -3
- package/dist/types/tools/toolbox_toolset.d.ts +15 -20
- package/dist/types/utils/async_queue.d.ts +14 -0
- package/dist/types/utils/env_aware_utils.d.ts +11 -0
- package/dist/types/utils/error_utils.d.ts +4 -0
- package/dist/types/utils/gemini_schema_util.d.ts +27 -6
- package/dist/types/utils/id_provider.d.ts +43 -0
- package/dist/types/utils/model_name.d.ts +8 -0
- package/dist/types/utils/ssrf_guard.d.ts +2 -0
- package/dist/types/version.d.ts +1 -1
- package/dist/types/workflow/index.d.ts +1 -1
- package/dist/types/workflow/nodes/parallel_worker.d.ts +8 -1
- package/dist/web/index_web.js +86 -88
- package/dist/web/index_web.js.map +4 -4
- package/package.json +13 -15
- package/dist/cjs/telemetry/google_cloud.js +0 -116
- package/dist/cjs/telemetry/tracing.js +0 -393
- package/dist/esm/telemetry/google_cloud.js +0 -99
- package/dist/esm/telemetry/setup.js +0 -88
- package/dist/esm/telemetry/tracing.js +0 -361
- package/dist/types/telemetry/google_cloud.d.ts +0 -9
- package/dist/types/telemetry/tracing.d.ts +0 -143
|
@@ -187,7 +187,9 @@ export declare function actionRequest(action: string, options?: {
|
|
|
187
187
|
/** Builds the response schema of an action. */
|
|
188
188
|
export declare function actionResponse(action: string): JsonObject;
|
|
189
189
|
/** Builds the request schema of the `ExecuteCustomQuery` action. */
|
|
190
|
-
export declare function executeCustomQueryRequest(
|
|
190
|
+
export declare function executeCustomQueryRequest(options?: {
|
|
191
|
+
includeDynamicAuthConfig?: boolean;
|
|
192
|
+
}): JsonObject;
|
|
191
193
|
/**
|
|
192
194
|
* A client for the Integration Connectors API.
|
|
193
195
|
*
|
|
@@ -208,14 +210,16 @@ export declare class ConnectionsClient {
|
|
|
208
210
|
* Reads the JSON schema of an entity and the operations it supports.
|
|
209
211
|
*
|
|
210
212
|
* @param entity The entity name.
|
|
211
|
-
* @throws Error When the API starts no operation for the entity
|
|
213
|
+
* @throws Error When the API starts no operation for the entity, or the
|
|
214
|
+
* operation fails or times out.
|
|
212
215
|
*/
|
|
213
216
|
getEntitySchemaAndOperations(entity: string): Promise<EntitySchemaAndOperations>;
|
|
214
217
|
/**
|
|
215
218
|
* Reads the input and output JSON schemas of an action.
|
|
216
219
|
*
|
|
217
220
|
* @param action The action name.
|
|
218
|
-
* @throws Error When the API starts no operation for the action
|
|
221
|
+
* @throws Error When the API starts no operation for the action, or the
|
|
222
|
+
* operation fails or times out.
|
|
219
223
|
*/
|
|
220
224
|
getActionSchema(action: string): Promise<ActionSchema>;
|
|
221
225
|
/**
|
|
@@ -224,6 +228,11 @@ export declare class ConnectionsClient {
|
|
|
224
228
|
*/
|
|
225
229
|
connectorPayload(jsonSchema: JsonObject): JsonObject;
|
|
226
230
|
private executeApiCall;
|
|
227
|
-
/**
|
|
231
|
+
/**
|
|
232
|
+
* Polls a long-running operation until it reports `done`.
|
|
233
|
+
*
|
|
234
|
+
* @throws Error When the operation reports an `error` or does not finish
|
|
235
|
+
* within `MAX_OPERATION_POLL_ATTEMPTS` polls.
|
|
236
|
+
*/
|
|
228
237
|
private pollOperation;
|
|
229
238
|
}
|
|
@@ -29,7 +29,7 @@ export declare abstract class BaseToolset {
|
|
|
29
29
|
readonly toolFilter: ToolPredicate | string[];
|
|
30
30
|
readonly prefix?: string | undefined;
|
|
31
31
|
readonly [BASE_TOOLSET_SIGNATURE_SYMBOL] = true;
|
|
32
|
-
constructor(toolFilter
|
|
32
|
+
constructor(toolFilter?: ToolPredicate | string[], prefix?: string | undefined);
|
|
33
33
|
/**
|
|
34
34
|
* Returns the tools that should be exposed to LLM.
|
|
35
35
|
*
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Google LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { Context } from '../../agents/context.js';
|
|
7
|
+
/**
|
|
8
|
+
* The environment a {@link BaseComputer} operates in.
|
|
9
|
+
*/
|
|
10
|
+
export declare enum ComputerEnvironment {
|
|
11
|
+
/** Defaults to browser. */
|
|
12
|
+
ENVIRONMENT_UNSPECIFIED = "ENVIRONMENT_UNSPECIFIED",
|
|
13
|
+
/** Operates in a web browser. */
|
|
14
|
+
ENVIRONMENT_BROWSER = "ENVIRONMENT_BROWSER"
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The current state of the computer environment, returned by every
|
|
18
|
+
* {@link BaseComputer} action.
|
|
19
|
+
*/
|
|
20
|
+
export interface ComputerState {
|
|
21
|
+
/** The screenshot in PNG format. */
|
|
22
|
+
screenshot?: Uint8Array;
|
|
23
|
+
/** The current URL of the webpage being displayed. */
|
|
24
|
+
url?: string;
|
|
25
|
+
}
|
|
26
|
+
/** A direction to scroll in. */
|
|
27
|
+
export type ScrollDirection = 'up' | 'down' | 'left' | 'right';
|
|
28
|
+
/**
|
|
29
|
+
* Throws when the computer use feature is turned off.
|
|
30
|
+
*
|
|
31
|
+
* @throws {Error} If `ADK_DISABLE_COMPUTER_USE` or a feature override turned
|
|
32
|
+
* the feature off.
|
|
33
|
+
*/
|
|
34
|
+
export declare function checkComputerUseEnabled(): void;
|
|
35
|
+
declare const BASE_COMPUTER_SIGNATURE_SYMBOL: unique symbol;
|
|
36
|
+
/**
|
|
37
|
+
* Type guard to check if an object is an instance of {@link BaseComputer}.
|
|
38
|
+
*
|
|
39
|
+
* @param obj The object to check.
|
|
40
|
+
* @returns True if the object is an instance of BaseComputer, false otherwise.
|
|
41
|
+
*/
|
|
42
|
+
export declare function isBaseComputer(obj: unknown): obj is BaseComputer;
|
|
43
|
+
/**
|
|
44
|
+
* Defines an interface for computer environments.
|
|
45
|
+
*
|
|
46
|
+
* This abstract base class defines the standard interface for controlling
|
|
47
|
+
* computer environments, including web browsers and other interactive systems.
|
|
48
|
+
* Coordinates passed to the action methods are absolute pixel values, scaled
|
|
49
|
+
* to the width and height returned by {@link screenSize}.
|
|
50
|
+
*
|
|
51
|
+
* The optional methods (`doubleClick`, `tripleClick`, `middleClick`,
|
|
52
|
+
* `rightClick`, `mouseDown`, `mouseUp`, `keyDown` and `keyUp`) are exposed to
|
|
53
|
+
* the model only when a subclass implements them.
|
|
54
|
+
*
|
|
55
|
+
* The Gemini 3.x actions `click`, `move`, `type`, `scroll`, `take_screenshot`,
|
|
56
|
+
* `press_key` and `hotkey` are always exposed. Each one calls the subclass
|
|
57
|
+
* method of the same name (`click`, `move`, `type`, `scroll`,
|
|
58
|
+
* `takeScreenshot`, `pressKey`, `hotkey`) when it is defined, and otherwise
|
|
59
|
+
* calls the matching predefined method.
|
|
60
|
+
*/
|
|
61
|
+
export declare abstract class BaseComputer {
|
|
62
|
+
/** A unique symbol to identify ADK base computer class. */
|
|
63
|
+
readonly [BASE_COMPUTER_SIGNATURE_SYMBOL] = true;
|
|
64
|
+
constructor();
|
|
65
|
+
/** Initializes the computer. */
|
|
66
|
+
initialize(): Promise<void>;
|
|
67
|
+
/** Cleans up the resources of the computer. */
|
|
68
|
+
close(): Promise<void>;
|
|
69
|
+
/**
|
|
70
|
+
* Called before each tool invocation to prepare resources.
|
|
71
|
+
*
|
|
72
|
+
* Override this to set up session-level resources (a sandbox, tokens, and so
|
|
73
|
+
* on), using the state of `toolContext` for persistence across invocations.
|
|
74
|
+
*
|
|
75
|
+
* @param _toolContext The tool context with session state access.
|
|
76
|
+
*/
|
|
77
|
+
prepare(_toolContext: Context): Promise<void>;
|
|
78
|
+
/**
|
|
79
|
+
* Returns the screen size of the environment.
|
|
80
|
+
*
|
|
81
|
+
* @returns A tuple of (width, height) in pixels.
|
|
82
|
+
*/
|
|
83
|
+
abstract screenSize(): Promise<[number, number]>;
|
|
84
|
+
/** Returns the environment of the computer. */
|
|
85
|
+
abstract environment(): Promise<ComputerEnvironment>;
|
|
86
|
+
/**
|
|
87
|
+
* Opens the web browser.
|
|
88
|
+
*
|
|
89
|
+
* @returns The current state after opening the browser.
|
|
90
|
+
*/
|
|
91
|
+
abstract openWebBrowser(): Promise<ComputerState>;
|
|
92
|
+
/**
|
|
93
|
+
* Clicks at a specific x, y coordinate on the webpage.
|
|
94
|
+
*
|
|
95
|
+
* @param x The x-coordinate to click at.
|
|
96
|
+
* @param y The y-coordinate to click at.
|
|
97
|
+
* @returns The current state after clicking.
|
|
98
|
+
*/
|
|
99
|
+
abstract clickAt(x: number, y: number): Promise<ComputerState>;
|
|
100
|
+
/**
|
|
101
|
+
* Hovers at a specific x, y coordinate on the webpage.
|
|
102
|
+
*
|
|
103
|
+
* May be used to explore sub-menus that appear on hover.
|
|
104
|
+
*
|
|
105
|
+
* @param x The x-coordinate to hover at.
|
|
106
|
+
* @param y The y-coordinate to hover at.
|
|
107
|
+
* @returns The current state after hovering.
|
|
108
|
+
*/
|
|
109
|
+
abstract hoverAt(x: number, y: number): Promise<ComputerState>;
|
|
110
|
+
/**
|
|
111
|
+
* Types text at a specific x, y coordinate.
|
|
112
|
+
*
|
|
113
|
+
* @param x The x-coordinate to type at.
|
|
114
|
+
* @param y The y-coordinate to type at.
|
|
115
|
+
* @param text The text to type.
|
|
116
|
+
* @param pressEnter Whether to press ENTER after typing. The toolset passes
|
|
117
|
+
* `true` when the model does not say.
|
|
118
|
+
* @param clearBeforeTyping Whether to clear existing content before typing.
|
|
119
|
+
* The toolset passes `true` when the model does not say.
|
|
120
|
+
* @returns The current state after typing.
|
|
121
|
+
*/
|
|
122
|
+
abstract typeTextAt(x: number, y: number, text: string, pressEnter?: boolean, clearBeforeTyping?: boolean): Promise<ComputerState>;
|
|
123
|
+
/**
|
|
124
|
+
* Types text into the currently focused element.
|
|
125
|
+
*
|
|
126
|
+
* By default, this calls {@link keyCombination} once per character in `text`
|
|
127
|
+
* (and once for `"Enter"` when `pressEnter` is true). Because
|
|
128
|
+
* `keyCombination` is designed for key names and typically captures a new
|
|
129
|
+
* screenshot on every call, this fallback costs one round trip and screenshot
|
|
130
|
+
* per character, and relies on the backend accepting single characters in
|
|
131
|
+
* `keyCombination`. Subclasses should override this method to use native
|
|
132
|
+
* bulk text input.
|
|
133
|
+
*
|
|
134
|
+
* @param text The text to type.
|
|
135
|
+
* @param pressEnter Whether to press ENTER after typing.
|
|
136
|
+
* @returns The current state after typing.
|
|
137
|
+
*/
|
|
138
|
+
typeText(text: string, pressEnter?: boolean): Promise<ComputerState>;
|
|
139
|
+
/**
|
|
140
|
+
* Scrolls the entire webpage up, down, left or right.
|
|
141
|
+
*
|
|
142
|
+
* @param direction The direction to scroll.
|
|
143
|
+
* @returns The current state after scrolling.
|
|
144
|
+
*/
|
|
145
|
+
abstract scrollDocument(direction: ScrollDirection): Promise<ComputerState>;
|
|
146
|
+
/**
|
|
147
|
+
* Scrolls up, down, right, or left at an x, y coordinate by magnitude.
|
|
148
|
+
*
|
|
149
|
+
* @param x The x-coordinate to scroll at.
|
|
150
|
+
* @param y The y-coordinate to scroll at.
|
|
151
|
+
* @param direction The direction to scroll.
|
|
152
|
+
* @param magnitude The amount to scroll.
|
|
153
|
+
* @returns The current state after scrolling.
|
|
154
|
+
*/
|
|
155
|
+
abstract scrollAt(x: number, y: number, direction: ScrollDirection, magnitude: number): Promise<ComputerState>;
|
|
156
|
+
/**
|
|
157
|
+
* Waits for n seconds to allow unfinished webpage processes to complete.
|
|
158
|
+
*
|
|
159
|
+
* @param seconds The number of seconds to wait.
|
|
160
|
+
* @returns The current state after waiting.
|
|
161
|
+
*/
|
|
162
|
+
abstract wait(seconds: number): Promise<ComputerState>;
|
|
163
|
+
/**
|
|
164
|
+
* Navigates back to the previous webpage in the browser history.
|
|
165
|
+
*
|
|
166
|
+
* @returns The current state after navigating back.
|
|
167
|
+
*/
|
|
168
|
+
abstract goBack(): Promise<ComputerState>;
|
|
169
|
+
/**
|
|
170
|
+
* Navigates forward to the next webpage in the browser history.
|
|
171
|
+
*
|
|
172
|
+
* @returns The current state after navigating forward.
|
|
173
|
+
*/
|
|
174
|
+
abstract goForward(): Promise<ComputerState>;
|
|
175
|
+
/**
|
|
176
|
+
* Directly jumps to a search engine home page.
|
|
177
|
+
*
|
|
178
|
+
* Used when you need to start with a search, for example because the current
|
|
179
|
+
* website doesn't have the information needed or a new task is being
|
|
180
|
+
* started.
|
|
181
|
+
*
|
|
182
|
+
* @returns The current state after navigating to search.
|
|
183
|
+
*/
|
|
184
|
+
abstract search(): Promise<ComputerState>;
|
|
185
|
+
/**
|
|
186
|
+
* Navigates directly to a specified URL.
|
|
187
|
+
*
|
|
188
|
+
* @param url The URL to navigate to.
|
|
189
|
+
* @returns The current state after navigation.
|
|
190
|
+
*/
|
|
191
|
+
abstract navigate(url: string): Promise<ComputerState>;
|
|
192
|
+
/**
|
|
193
|
+
* Presses keyboard keys and combinations, such as "control+c" or "enter".
|
|
194
|
+
*
|
|
195
|
+
* @param keys The keys to press in combination.
|
|
196
|
+
* @returns The current state after the key press.
|
|
197
|
+
*/
|
|
198
|
+
abstract keyCombination(keys: string[]): Promise<ComputerState>;
|
|
199
|
+
/**
|
|
200
|
+
* Drags an element from an x, y coordinate and drops it at a destination
|
|
201
|
+
* coordinate.
|
|
202
|
+
*
|
|
203
|
+
* @param x The x-coordinate to start dragging from.
|
|
204
|
+
* @param y The y-coordinate to start dragging from.
|
|
205
|
+
* @param destinationX The x-coordinate to drop at.
|
|
206
|
+
* @param destinationY The y-coordinate to drop at.
|
|
207
|
+
* @returns The current state after drag and drop.
|
|
208
|
+
*/
|
|
209
|
+
abstract dragAndDrop(x: number, y: number, destinationX: number, destinationY: number): Promise<ComputerState>;
|
|
210
|
+
/**
|
|
211
|
+
* Returns the current state of the current webpage.
|
|
212
|
+
*
|
|
213
|
+
* @returns The current environment state.
|
|
214
|
+
*/
|
|
215
|
+
abstract currentState(): Promise<ComputerState>;
|
|
216
|
+
/** Double-clicks at a specific x, y coordinate. */
|
|
217
|
+
doubleClick?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
218
|
+
/** Triple-clicks at a specific x, y coordinate. */
|
|
219
|
+
tripleClick?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
220
|
+
/** Clicks the middle mouse button at a specific x, y coordinate. */
|
|
221
|
+
middleClick?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
222
|
+
/** Clicks the right mouse button at a specific x, y coordinate. */
|
|
223
|
+
rightClick?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
224
|
+
/** Presses a mouse button down at a specific x, y coordinate. */
|
|
225
|
+
mouseDown?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
226
|
+
/** Releases a mouse button at a specific x, y coordinate. */
|
|
227
|
+
mouseUp?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
228
|
+
/** Presses a keyboard key down and holds it. */
|
|
229
|
+
keyDown?(key: string): Promise<ComputerState>;
|
|
230
|
+
/** Releases a keyboard key. */
|
|
231
|
+
keyUp?(key: string): Promise<ComputerState>;
|
|
232
|
+
/**
|
|
233
|
+
* Clicks at a specific x, y coordinate with the given mouse button.
|
|
234
|
+
*
|
|
235
|
+
* When defined, the `click` action calls this instead of {@link clickAt}
|
|
236
|
+
* and passes the model's `button`.
|
|
237
|
+
*/
|
|
238
|
+
click?(x: number, y: number, button?: string): Promise<ComputerState>;
|
|
239
|
+
/**
|
|
240
|
+
* Moves the cursor to a specific x, y coordinate.
|
|
241
|
+
*
|
|
242
|
+
* When defined, the `move` action calls this instead of {@link hoverAt}.
|
|
243
|
+
*/
|
|
244
|
+
move?(x: number, y: number): Promise<ComputerState>;
|
|
245
|
+
/**
|
|
246
|
+
* Types text into the currently focused element.
|
|
247
|
+
*
|
|
248
|
+
* When defined, the `type` action calls this instead of {@link typeText}.
|
|
249
|
+
* `pressEnter` is `undefined` when the model does not say.
|
|
250
|
+
*/
|
|
251
|
+
type?(text: string, pressEnter?: boolean): Promise<ComputerState>;
|
|
252
|
+
/**
|
|
253
|
+
* Scrolls at a specific x, y coordinate by a number of pixels.
|
|
254
|
+
*
|
|
255
|
+
* When defined, the `scroll` action calls this instead of {@link scrollAt}.
|
|
256
|
+
* `magnitudeInPixels` is `undefined` when the model does not say.
|
|
257
|
+
*/
|
|
258
|
+
scroll?(x: number, y: number, direction: ScrollDirection, magnitudeInPixels?: number): Promise<ComputerState>;
|
|
259
|
+
/**
|
|
260
|
+
* Takes a screenshot of the current state.
|
|
261
|
+
*
|
|
262
|
+
* When defined, the `take_screenshot` action calls this instead of
|
|
263
|
+
* {@link currentState}.
|
|
264
|
+
*/
|
|
265
|
+
takeScreenshot?(): Promise<ComputerState>;
|
|
266
|
+
/**
|
|
267
|
+
* Presses a single keyboard key.
|
|
268
|
+
*
|
|
269
|
+
* When defined, the `press_key` action calls this instead of
|
|
270
|
+
* {@link keyCombination}.
|
|
271
|
+
*/
|
|
272
|
+
pressKey?(key: string): Promise<ComputerState>;
|
|
273
|
+
/**
|
|
274
|
+
* Presses a combination of keyboard keys simultaneously.
|
|
275
|
+
*
|
|
276
|
+
* When defined, the `hotkey` action calls this instead of
|
|
277
|
+
* {@link keyCombination}.
|
|
278
|
+
*/
|
|
279
|
+
hotkey?(keys: string[]): Promise<ComputerState>;
|
|
280
|
+
}
|
|
281
|
+
export {};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Google LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { Schema } from '@google/genai';
|
|
7
|
+
import { BaseComputer } from './base_computer.js';
|
|
8
|
+
/**
|
|
9
|
+
* A computer use action the model can call, before it is wrapped in a
|
|
10
|
+
* `ComputerUseTool`.
|
|
11
|
+
*/
|
|
12
|
+
export interface ComputerAction {
|
|
13
|
+
/** The predefined function name the model calls, such as `click_at`. */
|
|
14
|
+
readonly name: string;
|
|
15
|
+
/** The description of the action. */
|
|
16
|
+
readonly description: string;
|
|
17
|
+
/** The schema of the action's arguments. */
|
|
18
|
+
readonly parameters: Schema;
|
|
19
|
+
/** Runs the action on the computer with the model's arguments. */
|
|
20
|
+
execute(args: Record<string, unknown>): Promise<unknown>;
|
|
21
|
+
}
|
|
22
|
+
/** The options for {@link createComputerActions}. */
|
|
23
|
+
export interface ComputerActionsOptions {
|
|
24
|
+
/** The computer the actions run on. */
|
|
25
|
+
computer: BaseComputer;
|
|
26
|
+
/** Whether `navigate` may target private and loopback hosts. */
|
|
27
|
+
allowPrivateNetworkAccess: boolean;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The predefined actions that map one-to-one onto {@link BaseComputer}
|
|
31
|
+
* methods, in the order the toolset exposes them.
|
|
32
|
+
*/
|
|
33
|
+
export declare const LEGACY_ACTION_NAMES: readonly string[];
|
|
34
|
+
/** Gemini 3.x mouse actions that are exposed only when the computer has them. */
|
|
35
|
+
export declare const OPTIONAL_MOUSE_ACTIONS: readonly ["double_click", "triple_click", "middle_click", "right_click", "mouse_down", "mouse_up"];
|
|
36
|
+
/** Gemini 3.x key actions that are exposed only when the computer has them. */
|
|
37
|
+
export declare const OPTIONAL_KEY_ACTIONS: readonly ["key_down", "key_up"];
|
|
38
|
+
/**
|
|
39
|
+
* Returns the optional Gemini 3.x actions that `computer` does not implement.
|
|
40
|
+
*
|
|
41
|
+
* @param computer The computer to probe.
|
|
42
|
+
* @returns The action names, in their canonical order.
|
|
43
|
+
*/
|
|
44
|
+
export declare function unimplementedOptionalActions(computer: BaseComputer): string[];
|
|
45
|
+
/**
|
|
46
|
+
* Builds every action the computer supports: the predefined actions, the
|
|
47
|
+
* Gemini 3.x actions that translate onto them, and the optional Gemini 3.x
|
|
48
|
+
* actions the computer implements.
|
|
49
|
+
*
|
|
50
|
+
* @param options The computer and its configuration.
|
|
51
|
+
* @returns The actions, legacy ones first.
|
|
52
|
+
*/
|
|
53
|
+
export declare function createComputerActions(options: ComputerActionsOptions): ComputerAction[];
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Google LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { Schema } from '@google/genai';
|
|
7
|
+
import { Context } from '../../agents/context.js';
|
|
8
|
+
import { RunAsyncToolRequest, ToolProcessLlmRequest } from '../base_tool.js';
|
|
9
|
+
import { FunctionTool } from '../function_tool.js';
|
|
10
|
+
/**
|
|
11
|
+
* The function a {@link ComputerUseTool} wraps. It receives the model's
|
|
12
|
+
* arguments, with coordinates already scaled to the screen.
|
|
13
|
+
*/
|
|
14
|
+
export type ComputerUseFunction = (args: Record<string, unknown>, toolContext?: Context) => Promise<unknown> | unknown;
|
|
15
|
+
/**
|
|
16
|
+
* The options for creating a {@link ComputerUseTool}.
|
|
17
|
+
*/
|
|
18
|
+
export interface ComputerUseToolOptions {
|
|
19
|
+
/** The name of the computer use action, such as `click_at`. */
|
|
20
|
+
name: string;
|
|
21
|
+
/** The description of the action. */
|
|
22
|
+
description: string;
|
|
23
|
+
/** The schema of the action's arguments. */
|
|
24
|
+
parameters?: Schema;
|
|
25
|
+
/** The computer control function to wrap. */
|
|
26
|
+
execute: ComputerUseFunction;
|
|
27
|
+
/**
|
|
28
|
+
* The actual screen size as (width, height) in pixels. This represents the
|
|
29
|
+
* real dimensions of the target screen.
|
|
30
|
+
*/
|
|
31
|
+
screenSize: readonly [number, number];
|
|
32
|
+
/**
|
|
33
|
+
* The virtual coordinate space as (width, height) that the model uses to
|
|
34
|
+
* specify coordinates. Coordinates from the model are scaled from this space
|
|
35
|
+
* to `screenSize`. Defaults to (1000, 1000), so the model works with a
|
|
36
|
+
* 1000x1000 screen regardless of the actual screen dimensions.
|
|
37
|
+
*/
|
|
38
|
+
virtualScreenSize?: readonly [number, number];
|
|
39
|
+
}
|
|
40
|
+
declare const COMPUTER_USE_TOOL_SIGNATURE_SYMBOL: unique symbol;
|
|
41
|
+
/**
|
|
42
|
+
* Type guard to check if an object is an instance of {@link ComputerUseTool}.
|
|
43
|
+
*
|
|
44
|
+
* @param obj The object to check.
|
|
45
|
+
* @returns True if the object is an instance of ComputerUseTool, false
|
|
46
|
+
* otherwise.
|
|
47
|
+
*/
|
|
48
|
+
export declare function isComputerUseTool(obj: unknown): obj is ComputerUseTool;
|
|
49
|
+
/**
|
|
50
|
+
* A tool that wraps computer control functions for use with LLMs.
|
|
51
|
+
*
|
|
52
|
+
* This tool scales coordinates from a virtual coordinate space (by default
|
|
53
|
+
* 1000x1000) to the actual screen size. This lets the model work with a
|
|
54
|
+
* consistent coordinate system regardless of the actual screen dimensions,
|
|
55
|
+
* which makes its output more predictable.
|
|
56
|
+
*
|
|
57
|
+
* When the wrapped function returns a {@link ComputerState}, the tool returns
|
|
58
|
+
* `{image: {mimetype, data}, url}` with the screenshot base64-encoded; the
|
|
59
|
+
* agent then sends the image to the model as part of the function response.
|
|
60
|
+
*
|
|
61
|
+
* When the model marks a call with a `safety_decision` of
|
|
62
|
+
* `require_confirmation`, the tool asks the user to confirm before it runs and
|
|
63
|
+
* adds `safety_acknowledgement: 'true'` to the response once confirmed.
|
|
64
|
+
*/
|
|
65
|
+
export declare class ComputerUseTool extends FunctionTool<Schema> {
|
|
66
|
+
/** A unique symbol to identify ADK computer use tool class. */
|
|
67
|
+
readonly [COMPUTER_USE_TOOL_SIGNATURE_SYMBOL] = true;
|
|
68
|
+
/** The wrapped computer control function. */
|
|
69
|
+
readonly func: ComputerUseFunction;
|
|
70
|
+
/** The actual screen size as (width, height) in pixels. */
|
|
71
|
+
readonly screenSize: readonly [number, number];
|
|
72
|
+
/** The virtual coordinate space as (width, height). */
|
|
73
|
+
readonly virtualScreenSize: readonly [number, number];
|
|
74
|
+
/**
|
|
75
|
+
* @param options The options for the tool.
|
|
76
|
+
* @throws {Error} If `screenSize` or `virtualScreenSize` is not a pair of
|
|
77
|
+
* positive numbers.
|
|
78
|
+
*/
|
|
79
|
+
constructor(options: ComputerUseToolOptions);
|
|
80
|
+
/**
|
|
81
|
+
* Runs the computer control function with scaled coordinates.
|
|
82
|
+
*
|
|
83
|
+
* @param request The tool request containing arguments and tool context.
|
|
84
|
+
* @returns The function's result, with a {@link ComputerState} converted to
|
|
85
|
+
* a response carrying the base64-encoded screenshot.
|
|
86
|
+
* @throws {Error} If a coordinate is not numeric, or the function returns a
|
|
87
|
+
* state with no screenshot.
|
|
88
|
+
*/
|
|
89
|
+
runAsync({ args, toolContext, }: RunAsyncToolRequest): Promise<unknown>;
|
|
90
|
+
/**
|
|
91
|
+
* Whether this call needs user confirmation: when the model marked it with
|
|
92
|
+
* a `safety_decision` of `require_confirmation`, or when the base tool says
|
|
93
|
+
* so.
|
|
94
|
+
*
|
|
95
|
+
* @param args The arguments the tool would run with.
|
|
96
|
+
* @param toolContext The context of the call, when there is one.
|
|
97
|
+
* @returns Whether the call requires confirmation.
|
|
98
|
+
*/
|
|
99
|
+
checkRequireConfirmation(args: Record<string, unknown>, toolContext?: Context): Promise<boolean>;
|
|
100
|
+
/**
|
|
101
|
+
* Adds nothing to the request: {@link ComputerUseToolset} adds the single
|
|
102
|
+
* computer use configuration and registers its tools by name.
|
|
103
|
+
*/
|
|
104
|
+
processLlmRequest(_request: ToolProcessLlmRequest): Promise<void>;
|
|
105
|
+
/** Returns a copy of `args` with every coordinate scaled to the screen. */
|
|
106
|
+
private scaleCoordinates;
|
|
107
|
+
}
|
|
108
|
+
export {};
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2026 Google LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { Context } from '../../agents/context.js';
|
|
7
|
+
import { LlmRequest } from '../../models/llm_request.js';
|
|
8
|
+
import { BaseToolset } from '../base_toolset.js';
|
|
9
|
+
import { BaseComputer } from './base_computer.js';
|
|
10
|
+
import { ComputerUseFunction, ComputerUseTool } from './computer_use_tool.js';
|
|
11
|
+
/**
|
|
12
|
+
* A computer use function, as returned by the adapter passed to
|
|
13
|
+
* {@link adaptComputerUseTool}.
|
|
14
|
+
*/
|
|
15
|
+
export interface AdaptedComputerUseFunction {
|
|
16
|
+
/** The name of the new tool. */
|
|
17
|
+
name: string;
|
|
18
|
+
/** The description of the new tool. */
|
|
19
|
+
description: string;
|
|
20
|
+
/** The new computer use function. */
|
|
21
|
+
execute: ComputerUseFunction;
|
|
22
|
+
}
|
|
23
|
+
/** The parameters of {@link adaptComputerUseTool}. */
|
|
24
|
+
export interface AdaptComputerUseToolParams {
|
|
25
|
+
/** The predefined action to adapt, such as `wait`. */
|
|
26
|
+
methodName: string;
|
|
27
|
+
/**
|
|
28
|
+
* Receives the original computer use function and returns the new one. It
|
|
29
|
+
* may be async.
|
|
30
|
+
*/
|
|
31
|
+
adapter: (func: ComputerUseFunction) => AdaptedComputerUseFunction | Promise<AdaptedComputerUseFunction>;
|
|
32
|
+
/** The request holding the tools. */
|
|
33
|
+
llmRequest: LlmRequest;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Replaces a computer use tool in the request with an adapted version.
|
|
37
|
+
*
|
|
38
|
+
* The adapted tool keeps the original tool's screen size and virtual screen
|
|
39
|
+
* size. When the adapted function has a new name, the original tool is
|
|
40
|
+
* removed from the request.
|
|
41
|
+
*
|
|
42
|
+
* @param params The action to adapt, the adapter and the request.
|
|
43
|
+
*/
|
|
44
|
+
export declare function adaptComputerUseTool({ methodName, adapter, llmRequest, }: AdaptComputerUseToolParams): Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* The options for creating a {@link ComputerUseToolset}.
|
|
47
|
+
*/
|
|
48
|
+
export interface ComputerUseToolsetOptions {
|
|
49
|
+
/** The computer environment to expose as tools. */
|
|
50
|
+
computer: BaseComputer;
|
|
51
|
+
/**
|
|
52
|
+
* Names of computer use actions that should not be exposed as tools. Both
|
|
53
|
+
* legacy names (such as `click_at`) and Gemini 3.x names (such as `click`)
|
|
54
|
+
* are accepted. Each is expanded to its counterpart locally, and translated
|
|
55
|
+
* to the target model's action schema in the request.
|
|
56
|
+
*/
|
|
57
|
+
excludedPredefinedFunctions?: readonly string[];
|
|
58
|
+
/**
|
|
59
|
+
* By default `navigate` refuses urls whose host is not publicly routable.
|
|
60
|
+
* Set this to true when the agent is meant to drive the browser against
|
|
61
|
+
* localhost or an internal host.
|
|
62
|
+
*/
|
|
63
|
+
allowPrivateNetworkAccess?: boolean;
|
|
64
|
+
}
|
|
65
|
+
declare const COMPUTER_USE_TOOLSET_SIGNATURE_SYMBOL: unique symbol;
|
|
66
|
+
/**
|
|
67
|
+
* Type guard to check if an object is an instance of
|
|
68
|
+
* {@link ComputerUseToolset}.
|
|
69
|
+
*
|
|
70
|
+
* @param obj The object to check.
|
|
71
|
+
* @returns True if the object is an instance of ComputerUseToolset, false
|
|
72
|
+
* otherwise.
|
|
73
|
+
*/
|
|
74
|
+
export declare function isComputerUseToolset(obj: unknown): obj is ComputerUseToolset;
|
|
75
|
+
/**
|
|
76
|
+
* A toolset that exposes the actions of a {@link BaseComputer} to an agent.
|
|
77
|
+
*
|
|
78
|
+
* The toolset initializes the computer on first use and wraps each action in a
|
|
79
|
+
* {@link ComputerUseTool}. Actions on one computer run one at a time, and the
|
|
80
|
+
* computer's `prepare()` runs before each of them. When the agent builds a
|
|
81
|
+
* request, the toolset registers its tools and adds the computer use
|
|
82
|
+
* configuration that tells the model which predefined actions it may call.
|
|
83
|
+
*
|
|
84
|
+
* ```ts
|
|
85
|
+
* const agent = new LlmAgent({
|
|
86
|
+
* name: 'browser_agent',
|
|
87
|
+
* model: 'gemini-flash-latest',
|
|
88
|
+
* tools: [new ComputerUseToolset({computer: new MyBrowserComputer()})],
|
|
89
|
+
* });
|
|
90
|
+
* ```
|
|
91
|
+
*/
|
|
92
|
+
export declare class ComputerUseToolset extends BaseToolset {
|
|
93
|
+
/** A unique symbol to identify ADK computer use toolset class. */
|
|
94
|
+
readonly [COMPUTER_USE_TOOLSET_SIGNATURE_SYMBOL] = true;
|
|
95
|
+
private readonly computer;
|
|
96
|
+
private readonly excludedPredefinedFunctions?;
|
|
97
|
+
private readonly allowPrivateNetworkAccess;
|
|
98
|
+
private initializing?;
|
|
99
|
+
private loadingTools?;
|
|
100
|
+
private closed;
|
|
101
|
+
/**
|
|
102
|
+
* @param options The computer and its configuration.
|
|
103
|
+
*/
|
|
104
|
+
constructor(options: ComputerUseToolsetOptions);
|
|
105
|
+
/**
|
|
106
|
+
* Returns a tool for every action the computer supports, except the
|
|
107
|
+
* excluded ones. The computer is initialized on the first call, and the
|
|
108
|
+
* tools are cached until {@link close}.
|
|
109
|
+
*/
|
|
110
|
+
getTools(): Promise<ComputerUseTool[]>;
|
|
111
|
+
/**
|
|
112
|
+
* Closes the computer and drops the cached tools. The next call to
|
|
113
|
+
* {@link getTools} initializes the computer again.
|
|
114
|
+
*/
|
|
115
|
+
close(): Promise<void>;
|
|
116
|
+
/**
|
|
117
|
+
* Registers the computer use tools in the request and adds the computer use
|
|
118
|
+
* configuration, unless the request already has one.
|
|
119
|
+
*
|
|
120
|
+
* @param _toolContext The context of the tool.
|
|
121
|
+
* @param llmRequest The outgoing request.
|
|
122
|
+
*/
|
|
123
|
+
processLlmRequest(_toolContext: Context, llmRequest: LlmRequest): Promise<void>;
|
|
124
|
+
private ensureInitialized;
|
|
125
|
+
private loadTools;
|
|
126
|
+
/**
|
|
127
|
+
* Wraps an action so it runs exclusively on the computer, after the
|
|
128
|
+
* computer has prepared for the calling session.
|
|
129
|
+
*/
|
|
130
|
+
private bindToComputer;
|
|
131
|
+
}
|
|
132
|
+
export {};
|
|
@@ -16,6 +16,7 @@ export interface GoogleApiToolsetOptions {
|
|
|
16
16
|
clientId?: string;
|
|
17
17
|
clientSecret?: string;
|
|
18
18
|
toolFilter?: ToolPredicate | string[];
|
|
19
|
+
scopes?: string[];
|
|
19
20
|
}
|
|
20
21
|
/**
|
|
21
22
|
* Type guard to check if an object is an instance of {@link GoogleApiToolset}.
|
|
@@ -30,6 +31,7 @@ export declare class GoogleApiToolset extends BaseToolset {
|
|
|
30
31
|
toolFilter: ToolPredicate | string[];
|
|
31
32
|
readonly apiName?: string;
|
|
32
33
|
readonly apiVersion?: string;
|
|
34
|
+
readonly scopes?: string[];
|
|
33
35
|
private clientId?;
|
|
34
36
|
private clientSecret?;
|
|
35
37
|
private openapiToolset?;
|
|
@@ -15,9 +15,13 @@ import { BaseTool, RunAsyncToolRequest, ToolProcessLlmRequest } from './base_too
|
|
|
15
15
|
*
|
|
16
16
|
* @param artifact The artifact to convert.
|
|
17
17
|
* @param artifactName The artifact name used in the placeholder text.
|
|
18
|
+
* @param options Optional settings controlling how inline media and file
|
|
19
|
+
* references are formatted.
|
|
18
20
|
* @return The artifact itself, or a text part that replaces it.
|
|
19
21
|
*/
|
|
20
|
-
export declare function asSafePartForLlm(artifact: Part, artifactName: string
|
|
22
|
+
export declare function asSafePartForLlm(artifact: Part, artifactName: string, options?: {
|
|
23
|
+
inlineMediaAsPlaceholder?: boolean;
|
|
24
|
+
}): Part;
|
|
21
25
|
/**
|
|
22
26
|
* A tool that loads the artifacts and adds them to the session.
|
|
23
27
|
*/
|