@signalwire/sdk 2.0.5 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +821 -0
- package/LICENSE +21 -0
- package/README.md +163 -68
- package/dist/AGENTS.md +13 -0
- package/dist/AgentBase.d.ts +510 -126
- package/dist/AgentBase.js +1581 -455
- package/dist/AgentBase.js.map +1 -1
- package/dist/AgentServer.d.ts +21 -0
- package/dist/AgentServer.js +154 -47
- package/dist/AgentServer.js.map +1 -1
- package/dist/AuthHandler.d.ts +14 -2
- package/dist/AuthHandler.js +7 -2
- package/dist/AuthHandler.js.map +1 -1
- package/dist/ConfigLoader.d.ts +5 -13
- package/dist/ConfigLoader.js +15 -29
- package/dist/ConfigLoader.js.map +1 -1
- package/dist/ContextBuilder.d.ts +202 -9
- package/dist/ContextBuilder.js +219 -36
- package/dist/ContextBuilder.js.map +1 -1
- package/dist/DataMap.d.ts +186 -45
- package/dist/DataMap.js +210 -52
- package/dist/DataMap.js.map +1 -1
- package/dist/FunctionResult.d.ts +309 -83
- package/dist/FunctionResult.js +483 -116
- package/dist/FunctionResult.js.map +1 -1
- package/dist/Logger.d.ts +30 -5
- package/dist/Logger.js +77 -19
- package/dist/Logger.js.map +1 -1
- package/dist/POM/PromptObjectModel.d.ts +12 -1
- package/dist/POM/PromptObjectModel.js +25 -9
- package/dist/POM/PromptObjectModel.js.map +1 -1
- package/dist/ParameterSchema.d.ts +306 -0
- package/dist/ParameterSchema.js +251 -0
- package/dist/ParameterSchema.js.map +1 -0
- package/dist/PlatformContracts.d.ts +21 -0
- package/dist/PlatformContracts.generated.d.ts +109 -0
- package/dist/PlatformContracts.generated.js +8 -0
- package/dist/PlatformContracts.generated.js.map +1 -0
- package/dist/PlatformContracts.js +2 -0
- package/dist/PlatformContracts.js.map +1 -0
- package/dist/PomBuilder.d.ts +18 -0
- package/dist/PomBuilder.js +53 -3
- package/dist/PomBuilder.js.map +1 -1
- package/dist/PostPrompt.d.ts +110 -0
- package/dist/PostPrompt.js +204 -0
- package/dist/PostPrompt.js.map +1 -0
- package/dist/PromptManager.d.ts +22 -1
- package/dist/PromptManager.js +28 -1
- package/dist/PromptManager.js.map +1 -1
- package/dist/PublicFetch.d.ts +132 -0
- package/dist/PublicFetch.js +311 -0
- package/dist/PublicFetch.js.map +1 -0
- package/dist/RobotsTxt.d.ts +41 -0
- package/dist/RobotsTxt.js +152 -0
- package/dist/RobotsTxt.js.map +1 -0
- package/dist/SWMLHandler.js.map +1 -1
- package/dist/SWMLService.d.ts +169 -59
- package/dist/SWMLService.js +456 -129
- package/dist/SWMLService.js.map +1 -1
- package/dist/SchemaUtils.d.ts +115 -2
- package/dist/SchemaUtils.js +444 -12
- package/dist/SchemaUtils.js.map +1 -1
- package/dist/SecurityUtils.d.ts +49 -9
- package/dist/SecurityUtils.js +153 -51
- package/dist/SecurityUtils.js.map +1 -1
- package/dist/SemanticGate.d.ts +183 -0
- package/dist/SemanticGate.js +533 -0
- package/dist/SemanticGate.js.map +1 -0
- package/dist/ServerlessAdapter.d.ts +156 -7
- package/dist/ServerlessAdapter.js +399 -42
- package/dist/ServerlessAdapter.js.map +1 -1
- package/dist/SessionManager.d.ts +11 -2
- package/dist/SessionManager.js +50 -18
- package/dist/SessionManager.js.map +1 -1
- package/dist/SslConfig.d.ts +2 -1
- package/dist/SslConfig.js +4 -1
- package/dist/SslConfig.js.map +1 -1
- package/dist/SwaigActions.generated.d.ts +23 -0
- package/dist/SwaigActions.generated.js +8 -0
- package/dist/SwaigActions.generated.js.map +1 -0
- package/dist/SwaigContracts.d.ts +15 -0
- package/dist/SwaigContracts.generated.d.ts +264 -0
- package/dist/SwaigContracts.generated.js +12 -0
- package/dist/SwaigContracts.generated.js.map +1 -0
- package/dist/SwaigContracts.js +2 -0
- package/dist/SwaigContracts.js.map +1 -0
- package/dist/SwaigFunction.d.ts +139 -7
- package/dist/SwaigFunction.js +170 -22
- package/dist/SwaigFunction.js.map +1 -1
- package/dist/SwmlBuilder.d.ts +68 -19
- package/dist/SwmlBuilder.js +74 -38
- package/dist/SwmlBuilder.js.map +1 -1
- package/dist/SwmlVerbMethods.generated.d.ts +41 -4
- package/dist/SwmlVerbMethods.generated.js +1 -1
- package/dist/TypeInference.d.ts +39 -3
- package/dist/TypeInference.js +256 -16
- package/dist/TypeInference.js.map +1 -1
- package/dist/WebService.d.ts +79 -11
- package/dist/WebService.js +320 -109
- package/dist/WebService.js.map +1 -1
- package/dist/WebhookMiddleware.d.ts +119 -0
- package/dist/WebhookMiddleware.js +271 -0
- package/dist/WebhookMiddleware.js.map +1 -0
- package/dist/WebhookValidator.d.ts +92 -0
- package/dist/WebhookValidator.js +409 -0
- package/dist/WebhookValidator.js.map +1 -0
- package/dist/agents/BedrockAgent.d.ts +162 -0
- package/dist/agents/BedrockAgent.js +323 -0
- package/dist/agents/BedrockAgent.js.map +1 -0
- package/dist/ai-chat/AIChatClient.d.ts +316 -0
- package/dist/ai-chat/AIChatClient.js +535 -0
- package/dist/ai-chat/AIChatClient.js.map +1 -0
- package/dist/ai-chat/ChatGateway.d.ts +408 -0
- package/dist/ai-chat/ChatGateway.js +747 -0
- package/dist/ai-chat/ChatGateway.js.map +1 -0
- package/dist/ai-chat/HandoffRouter.d.ts +282 -0
- package/dist/ai-chat/HandoffRouter.js +518 -0
- package/dist/ai-chat/HandoffRouter.js.map +1 -0
- package/dist/ai-chat/index.d.ts +12 -0
- package/dist/ai-chat/index.js +10 -0
- package/dist/ai-chat/index.js.map +1 -0
- package/dist/capabilities.d.ts +64 -0
- package/dist/capabilities.js +95 -0
- package/dist/capabilities.js.map +1 -0
- package/dist/cli/agent-loader.d.ts +32 -6
- package/dist/cli/agent-loader.js +92 -33
- package/dist/cli/agent-loader.js.map +1 -1
- package/dist/cli/datamap-exec.d.ts +175 -0
- package/dist/cli/datamap-exec.js +1284 -0
- package/dist/cli/datamap-exec.js.map +1 -0
- package/dist/cli/function-args.d.ts +40 -0
- package/dist/cli/function-args.js +146 -0
- package/dist/cli/function-args.js.map +1 -0
- package/dist/cli/simulation.d.ts +99 -0
- package/dist/cli/simulation.js +490 -0
- package/dist/cli/simulation.js.map +1 -0
- package/dist/cli/swaig-test.d.ts +12 -25
- package/dist/cli/swaig-test.js +935 -371
- package/dist/cli/swaig-test.js.map +1 -1
- package/dist/cli/tsdocs/agentsNote.d.ts +30 -0
- package/dist/cli/tsdocs/agentsNote.js +92 -0
- package/dist/cli/tsdocs/agentsNote.js.map +1 -0
- package/dist/cli/tsdocs/api-index.json +1 -0
- package/dist/cli/tsdocs/api.d.ts +91 -0
- package/dist/cli/tsdocs/api.js +372 -0
- package/dist/cli/tsdocs/api.js.map +1 -0
- package/dist/cli/tsdocs/apiTypes.d.ts +64 -0
- package/dist/cli/tsdocs/apiTypes.js +7 -0
- package/dist/cli/tsdocs/apiTypes.js.map +1 -0
- package/dist/cli/tsdocs/bin.d.ts +6 -0
- package/dist/cli/tsdocs/bin.js +23 -0
- package/dist/cli/tsdocs/bin.js.map +1 -0
- package/dist/cli/tsdocs/files.d.ts +108 -0
- package/dist/cli/tsdocs/files.js +287 -0
- package/dist/cli/tsdocs/files.js.map +1 -0
- package/dist/cli/tsdocs/index.d.ts +28 -0
- package/dist/cli/tsdocs/index.js +307 -0
- package/dist/cli/tsdocs/index.js.map +1 -0
- package/dist/cli/tsdocs/live.d.ts +41 -0
- package/dist/cli/tsdocs/live.js +202 -0
- package/dist/cli/tsdocs/live.js.map +1 -0
- package/dist/cli/tsdocs/locate.d.ts +51 -0
- package/dist/cli/tsdocs/locate.js +75 -0
- package/dist/cli/tsdocs/locate.js.map +1 -0
- package/dist/cli/tsdocs/render.d.ts +75 -0
- package/dist/cli/tsdocs/render.js +323 -0
- package/dist/cli/tsdocs/render.js.map +1 -0
- package/dist/cli/tsdocs/topics.d.ts +43 -0
- package/dist/cli/tsdocs/topics.js +920 -0
- package/dist/cli/tsdocs/topics.js.map +1 -0
- package/dist/index.d.ts +49 -68
- package/dist/index.js +45 -73
- package/dist/index.js.map +1 -1
- package/dist/livewire/index.d.ts +229 -155
- package/dist/livewire/index.js +298 -195
- package/dist/livewire/index.js.map +1 -1
- package/dist/llms.txt +65 -0
- package/dist/prefabs/CallSessionStore.d.ts +52 -0
- package/dist/prefabs/CallSessionStore.js +72 -0
- package/dist/prefabs/CallSessionStore.js.map +1 -0
- package/dist/prefabs/ConciergeAgent.d.ts +13 -1
- package/dist/prefabs/ConciergeAgent.js +36 -31
- package/dist/prefabs/ConciergeAgent.js.map +1 -1
- package/dist/prefabs/FAQBotAgent.d.ts +9 -1
- package/dist/prefabs/FAQBotAgent.js +113 -47
- package/dist/prefabs/FAQBotAgent.js.map +1 -1
- package/dist/prefabs/InfoGathererAgent.d.ts +22 -3
- package/dist/prefabs/InfoGathererAgent.js +67 -58
- package/dist/prefabs/InfoGathererAgent.js.map +1 -1
- package/dist/prefabs/ReceptionistAgent.d.ts +16 -5
- package/dist/prefabs/ReceptionistAgent.js +39 -31
- package/dist/prefabs/ReceptionistAgent.js.map +1 -1
- package/dist/prefabs/SurveyAgent.d.ts +22 -4
- package/dist/prefabs/SurveyAgent.js +84 -68
- package/dist/prefabs/SurveyAgent.js.map +1 -1
- package/dist/relay/Action.d.ts +44 -21
- package/dist/relay/Action.js +75 -17
- package/dist/relay/Action.js.map +1 -1
- package/dist/relay/Call.d.ts +312 -47
- package/dist/relay/Call.js +369 -18
- package/dist/relay/Call.js.map +1 -1
- package/dist/relay/Deferred.js.map +1 -1
- package/dist/relay/Message.d.ts +26 -5
- package/dist/relay/Message.js +36 -6
- package/dist/relay/Message.js.map +1 -1
- package/dist/relay/RelayClient.d.ts +44 -17
- package/dist/relay/RelayClient.js +175 -65
- package/dist/relay/RelayClient.js.map +1 -1
- package/dist/relay/RelayError.d.ts +14 -0
- package/dist/relay/RelayError.js +15 -0
- package/dist/relay/RelayError.js.map +1 -1
- package/dist/relay/RelayEvent.d.ts +96 -65
- package/dist/relay/RelayEvent.js +85 -56
- package/dist/relay/RelayEvent.js.map +1 -1
- package/dist/relay/closedSets.d.ts +111 -0
- package/dist/relay/closedSets.js +83 -0
- package/dist/relay/closedSets.js.map +1 -0
- package/dist/relay/index.d.ts +3 -1
- package/dist/relay/index.js +1 -0
- package/dist/relay/index.js.map +1 -1
- package/dist/relay/normalize.js.map +1 -1
- package/dist/relay/protocol.types.generated.d.ts +1195 -0
- package/dist/relay/protocol.types.generated.js +8 -0
- package/dist/relay/protocol.types.generated.js.map +1 -0
- package/dist/relay/types.d.ts +73 -42
- package/dist/rest/HttpClient.d.ts +36 -8
- package/dist/rest/HttpClient.js +233 -40
- package/dist/rest/HttpClient.js.map +1 -1
- package/dist/rest/RequestOptions.d.ts +110 -0
- package/dist/rest/RequestOptions.js +113 -0
- package/dist/rest/RequestOptions.js.map +1 -0
- package/dist/rest/RestError.d.ts +52 -6
- package/dist/rest/RestError.js +89 -5
- package/dist/rest/RestError.js.map +1 -1
- package/dist/rest/base/CrudResource.d.ts +16 -26
- package/dist/rest/base/CrudResource.js +19 -33
- package/dist/rest/base/CrudResource.js.map +1 -1
- package/dist/rest/base/CrudWithAddresses.d.ts +3 -2
- package/dist/rest/base/CrudWithAddresses.js +2 -2
- package/dist/rest/base/CrudWithAddresses.js.map +1 -1
- package/dist/rest/base/FabricResource.d.ts +15 -0
- package/dist/rest/base/FabricResource.js +16 -0
- package/dist/rest/base/FabricResource.js.map +1 -0
- package/dist/rest/base/ReadResource.d.ts +62 -0
- package/dist/rest/base/ReadResource.js +69 -0
- package/dist/rest/base/ReadResource.js.map +1 -0
- package/dist/rest/callHandler.d.ts +5 -2
- package/dist/rest/callHandler.js +4 -1
- package/dist/rest/callHandler.js.map +1 -1
- package/dist/rest/index.d.ts +18 -77
- package/dist/rest/index.js +47 -120
- package/dist/rest/index.js.map +1 -1
- package/dist/rest/namespaces/_client_tree_generated.d.ts +103 -0
- package/dist/rest/namespaces/_client_tree_generated.js +171 -0
- package/dist/rest/namespaces/_client_tree_generated.js.map +1 -0
- package/dist/rest/namespaces/addresses.d.ts +3 -42
- package/dist/rest/namespaces/addresses.js +3 -50
- package/dist/rest/namespaces/addresses.js.map +1 -1
- package/dist/rest/namespaces/calling.d.ts +6 -378
- package/dist/rest/namespaces/calling.js +6 -473
- package/dist/rest/namespaces/calling.js.map +1 -1
- package/dist/rest/namespaces/calling.resources.generated.d.ts +165 -0
- package/dist/rest/namespaces/calling.resources.generated.js +477 -0
- package/dist/rest/namespaces/calling.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/calling.types.generated.d.ts +2917 -0
- package/dist/rest/namespaces/calling.types.generated.js +8 -0
- package/dist/rest/namespaces/calling.types.generated.js.map +1 -0
- package/dist/rest/namespaces/chat.d.ts +4 -18
- package/dist/rest/namespaces/chat.js +4 -21
- package/dist/rest/namespaces/chat.js.map +1 -1
- package/dist/rest/namespaces/chat.resources.generated.d.ts +12 -0
- package/dist/rest/namespaces/chat.resources.generated.js +29 -0
- package/dist/rest/namespaces/chat.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/chat.types.generated.d.ts +56 -0
- package/dist/rest/namespaces/chat.types.generated.js +8 -0
- package/dist/rest/namespaces/chat.types.generated.js.map +1 -0
- package/dist/rest/namespaces/datasphere.d.ts +6 -62
- package/dist/rest/namespaces/datasphere.js +6 -72
- package/dist/rest/namespaces/datasphere.js.map +1 -1
- package/dist/rest/namespaces/datasphere.resources.generated.d.ts +25 -0
- package/dist/rest/namespaces/datasphere.resources.generated.js +50 -0
- package/dist/rest/namespaces/datasphere.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/datasphere.types.generated.d.ts +180 -0
- package/dist/rest/namespaces/datasphere.types.generated.js +8 -0
- package/dist/rest/namespaces/datasphere.types.generated.js.map +1 -0
- package/dist/rest/namespaces/fabric.d.ts +11 -327
- package/dist/rest/namespaces/fabric.js +14 -439
- package/dist/rest/namespaces/fabric.js.map +1 -1
- package/dist/rest/namespaces/fabric.resources.generated.d.ts +195 -0
- package/dist/rest/namespaces/fabric.resources.generated.js +399 -0
- package/dist/rest/namespaces/fabric.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/fabric.types.generated.d.ts +4557 -0
- package/dist/rest/namespaces/fabric.types.generated.js +8 -0
- package/dist/rest/namespaces/fabric.types.generated.js.map +1 -0
- package/dist/rest/namespaces/fax.resources.generated.d.ts +6 -0
- package/dist/rest/namespaces/fax.resources.generated.js +14 -0
- package/dist/rest/namespaces/fax.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/fax.types.generated.d.ts +131 -0
- package/dist/rest/namespaces/fax.types.generated.js +8 -0
- package/dist/rest/namespaces/fax.types.generated.js.map +1 -0
- package/dist/rest/namespaces/imported-numbers.d.ts +3 -18
- package/dist/rest/namespaces/imported-numbers.js +3 -21
- package/dist/rest/namespaces/imported-numbers.js.map +1 -1
- package/dist/rest/namespaces/logs.d.ts +10 -101
- package/dist/rest/namespaces/logs.js +10 -128
- package/dist/rest/namespaces/logs.js.map +1 -1
- package/dist/rest/namespaces/logs.resources.generated.d.ts +9 -0
- package/dist/rest/namespaces/logs.resources.generated.js +17 -0
- package/dist/rest/namespaces/logs.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/logs.types.generated.d.ts +131 -0
- package/dist/rest/namespaces/logs.types.generated.js +8 -0
- package/dist/rest/namespaces/logs.types.generated.js.map +1 -0
- package/dist/rest/namespaces/lookup.d.ts +3 -25
- package/dist/rest/namespaces/lookup.js +3 -27
- package/dist/rest/namespaces/lookup.js.map +1 -1
- package/dist/rest/namespaces/message.resources.generated.d.ts +6 -0
- package/dist/rest/namespaces/message.resources.generated.js +14 -0
- package/dist/rest/namespaces/message.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/message.types.generated.d.ts +122 -0
- package/dist/rest/namespaces/message.types.generated.js +8 -0
- package/dist/rest/namespaces/message.types.generated.js.map +1 -0
- package/dist/rest/namespaces/messages.resources.generated.d.ts +18 -0
- package/dist/rest/namespaces/messages.resources.generated.js +44 -0
- package/dist/rest/namespaces/messages.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/messages.types.generated.d.ts +99 -0
- package/dist/rest/namespaces/messages.types.generated.js +8 -0
- package/dist/rest/namespaces/messages.types.generated.js.map +1 -0
- package/dist/rest/namespaces/mfa.d.ts +3 -42
- package/dist/rest/namespaces/mfa.js +3 -49
- package/dist/rest/namespaces/mfa.js.map +1 -1
- package/dist/rest/namespaces/number-groups.d.ts +3 -45
- package/dist/rest/namespaces/number-groups.js +3 -53
- package/dist/rest/namespaces/number-groups.js.map +1 -1
- package/dist/rest/namespaces/phone-numbers.d.ts +8 -196
- package/dist/rest/namespaces/phone-numbers.js +8 -230
- package/dist/rest/namespaces/phone-numbers.js.map +1 -1
- package/dist/rest/namespaces/project.d.ts +6 -42
- package/dist/rest/namespaces/project.js +6 -51
- package/dist/rest/namespaces/project.js.map +1 -1
- package/dist/rest/namespaces/project.resources.generated.d.ts +17 -0
- package/dist/rest/namespaces/project.resources.generated.js +44 -0
- package/dist/rest/namespaces/project.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/project.types.generated.d.ts +58 -0
- package/dist/rest/namespaces/project.types.generated.js +8 -0
- package/dist/rest/namespaces/project.types.generated.js.map +1 -0
- package/dist/rest/namespaces/projects.resources.generated.d.ts +12 -0
- package/dist/rest/namespaces/projects.resources.generated.js +25 -0
- package/dist/rest/namespaces/projects.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/projects.types.generated.d.ts +77 -0
- package/dist/rest/namespaces/projects.types.generated.js +8 -0
- package/dist/rest/namespaces/projects.types.generated.js.map +1 -0
- package/dist/rest/namespaces/pubsub.d.ts +4 -18
- package/dist/rest/namespaces/pubsub.js +4 -21
- package/dist/rest/namespaces/pubsub.js.map +1 -1
- package/dist/rest/namespaces/pubsub.resources.generated.d.ts +12 -0
- package/dist/rest/namespaces/pubsub.resources.generated.js +29 -0
- package/dist/rest/namespaces/pubsub.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/pubsub.types.generated.d.ts +60 -0
- package/dist/rest/namespaces/pubsub.types.generated.js +8 -0
- package/dist/rest/namespaces/pubsub.types.generated.js.map +1 -0
- package/dist/rest/namespaces/queues.d.ts +3 -38
- package/dist/rest/namespaces/queues.js +3 -44
- package/dist/rest/namespaces/queues.js.map +1 -1
- package/dist/rest/namespaces/recordings.d.ts +3 -34
- package/dist/rest/namespaces/recordings.js +3 -40
- package/dist/rest/namespaces/recordings.js.map +1 -1
- package/dist/rest/namespaces/registry.d.ts +6 -137
- package/dist/rest/namespaces/registry.js +6 -173
- package/dist/rest/namespaces/registry.js.map +1 -1
- package/dist/rest/namespaces/relay-rest.resources.generated.d.ts +168 -0
- package/dist/rest/namespaces/relay-rest.resources.generated.js +430 -0
- package/dist/rest/namespaces/relay-rest.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/relay-rest.types.generated.d.ts +1906 -0
- package/dist/rest/namespaces/relay-rest.types.generated.js +8 -0
- package/dist/rest/namespaces/relay-rest.types.generated.js.map +1 -0
- package/dist/rest/namespaces/short-codes.d.ts +3 -35
- package/dist/rest/namespaces/short-codes.js +3 -41
- package/dist/rest/namespaces/short-codes.js.map +1 -1
- package/dist/rest/namespaces/sip-profile.d.ts +3 -24
- package/dist/rest/namespaces/sip-profile.js +3 -29
- package/dist/rest/namespaces/sip-profile.js.map +1 -1
- package/dist/rest/namespaces/verified-callers.d.ts +3 -29
- package/dist/rest/namespaces/verified-callers.js +3 -34
- package/dist/rest/namespaces/verified-callers.js.map +1 -1
- package/dist/rest/namespaces/video.d.ts +10 -227
- package/dist/rest/namespaces/video.js +10 -288
- package/dist/rest/namespaces/video.js.map +1 -1
- package/dist/rest/namespaces/video.resources.generated.d.ts +81 -0
- package/dist/rest/namespaces/video.resources.generated.js +171 -0
- package/dist/rest/namespaces/video.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/video.types.generated.d.ts +771 -0
- package/dist/rest/namespaces/video.types.generated.js +8 -0
- package/dist/rest/namespaces/video.types.generated.js.map +1 -0
- package/dist/rest/namespaces/voice.resources.generated.d.ts +9 -0
- package/dist/rest/namespaces/voice.resources.generated.js +17 -0
- package/dist/rest/namespaces/voice.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/voice.types.generated.d.ts +214 -0
- package/dist/rest/namespaces/voice.types.generated.js +8 -0
- package/dist/rest/namespaces/voice.types.generated.js.map +1 -0
- package/dist/rest/pagination.d.ts +5 -1
- package/dist/rest/pagination.js +40 -20
- package/dist/rest/pagination.js.map +1 -1
- package/dist/rest/types.d.ts +21 -3
- package/dist/schema.json +299 -98
- package/dist/skills/SkillBase.d.ts +111 -21
- package/dist/skills/SkillBase.js +100 -17
- package/dist/skills/SkillBase.js.map +1 -1
- package/dist/skills/SkillManager.d.ts +66 -13
- package/dist/skills/SkillManager.js +101 -20
- package/dist/skills/SkillManager.js.map +1 -1
- package/dist/skills/SkillName.d.ts +44 -0
- package/dist/skills/SkillName.js +2 -0
- package/dist/skills/SkillName.js.map +1 -0
- package/dist/skills/SkillRegistry.d.ts +22 -11
- package/dist/skills/SkillRegistry.js +47 -17
- package/dist/skills/SkillRegistry.js.map +1 -1
- package/dist/skills/builtin/api_ninjas_trivia.d.ts +3 -1
- package/dist/skills/builtin/api_ninjas_trivia.js +18 -6
- package/dist/skills/builtin/api_ninjas_trivia.js.map +1 -1
- package/dist/skills/builtin/ask_claude.d.ts +14 -4
- package/dist/skills/builtin/ask_claude.js +36 -14
- package/dist/skills/builtin/ask_claude.js.map +1 -1
- package/dist/skills/builtin/claude_skills.d.ts +3 -1
- package/dist/skills/builtin/claude_skills.js +19 -15
- package/dist/skills/builtin/claude_skills.js.map +1 -1
- package/dist/skills/builtin/custom_skills.d.ts +10 -3
- package/dist/skills/builtin/custom_skills.js +26 -9
- package/dist/skills/builtin/custom_skills.js.map +1 -1
- package/dist/skills/builtin/datasphere.d.ts +5 -3
- package/dist/skills/builtin/datasphere.js +21 -22
- package/dist/skills/builtin/datasphere.js.map +1 -1
- package/dist/skills/builtin/datasphere_serverless.d.ts +12 -1
- package/dist/skills/builtin/datasphere_serverless.js +17 -6
- package/dist/skills/builtin/datasphere_serverless.js.map +1 -1
- package/dist/skills/builtin/datetime.d.ts +3 -1
- package/dist/skills/builtin/datetime.js +11 -7
- package/dist/skills/builtin/datetime.js.map +1 -1
- package/dist/skills/builtin/google_maps.d.ts +29 -14
- package/dist/skills/builtin/google_maps.js +89 -269
- package/dist/skills/builtin/google_maps.js.map +1 -1
- package/dist/skills/builtin/index.d.ts +8 -6
- package/dist/skills/builtin/index.js +8 -6
- package/dist/skills/builtin/index.js.map +1 -1
- package/dist/skills/builtin/info_gatherer.d.ts +4 -2
- package/dist/skills/builtin/info_gatherer.js +20 -12
- package/dist/skills/builtin/info_gatherer.js.map +1 -1
- package/dist/skills/builtin/joke.d.ts +16 -6
- package/dist/skills/builtin/joke.js +54 -37
- package/dist/skills/builtin/joke.js.map +1 -1
- package/dist/skills/builtin/math.d.ts +3 -1
- package/dist/skills/builtin/math.js +14 -9
- package/dist/skills/builtin/math.js.map +1 -1
- package/dist/skills/builtin/mcp_gateway.d.ts +34 -5
- package/dist/skills/builtin/mcp_gateway.js +77 -46
- package/dist/skills/builtin/mcp_gateway.js.map +1 -1
- package/dist/skills/builtin/native_vector_search.d.ts +19 -5
- package/dist/skills/builtin/native_vector_search.js +179 -61
- package/dist/skills/builtin/native_vector_search.js.map +1 -1
- package/dist/skills/builtin/play_background_file.d.ts +21 -15
- package/dist/skills/builtin/play_background_file.js +43 -122
- package/dist/skills/builtin/play_background_file.js.map +1 -1
- package/dist/skills/builtin/spider.d.ts +42 -4
- package/dist/skills/builtin/spider.js +277 -104
- package/dist/skills/builtin/spider.js.map +1 -1
- package/dist/skills/builtin/swml_transfer.d.ts +14 -3
- package/dist/skills/builtin/swml_transfer.js +45 -24
- package/dist/skills/builtin/swml_transfer.js.map +1 -1
- package/dist/skills/builtin/weather_api.d.ts +24 -13
- package/dist/skills/builtin/weather_api.js +38 -30
- package/dist/skills/builtin/weather_api.js.map +1 -1
- package/dist/skills/builtin/web_search.d.ts +48 -10
- package/dist/skills/builtin/web_search.js +294 -100
- package/dist/skills/builtin/web_search.js.map +1 -1
- package/dist/skills/builtin/wikipedia_search.d.ts +4 -2
- package/dist/skills/builtin/wikipedia_search.js +14 -13
- package/dist/skills/builtin/wikipedia_search.js.map +1 -1
- package/dist/skills/index.d.ts +3 -2
- package/dist/skills/index.js +1 -1
- package/dist/skills/index.js.map +1 -1
- package/dist/swml_verbs_generated.d.ts +2824 -0
- package/dist/swml_verbs_generated.js +12 -0
- package/dist/swml_verbs_generated.js.map +1 -0
- package/dist/types.d.ts +59 -6
- package/dist/web.d.ts +27 -0
- package/dist/web.js +20 -0
- package/dist/web.js.map +1 -0
- package/docs/agent-guide.md +1034 -0
- package/docs/ai_chat.md +560 -0
- package/docs/api-reference.md +3641 -0
- package/docs/architecture.md +846 -0
- package/docs/bedrock_agent.md +411 -0
- package/docs/cli-guide.md +403 -0
- package/docs/cloud_functions_guide.md +399 -0
- package/docs/configuration.md +548 -0
- package/docs/contexts-guide.md +884 -0
- package/docs/datamap-guide.md +1207 -0
- package/docs/developer_pain_points.md +548 -0
- package/docs/llm_parameters.md +298 -0
- package/docs/mcp_gateway_reference.md +354 -0
- package/docs/mcp_integration.md +162 -0
- package/docs/pgi_agent_guide.md +1820 -0
- package/docs/prefabs-guide.md +815 -0
- package/docs/programmatically_governed_inference.md +301 -0
- package/docs/sdk_features.md +501 -0
- package/docs/security.md +559 -0
- package/docs/serverless-guide.md +520 -0
- package/docs/skills-guide.md +1223 -0
- package/docs/skills-system.md +405 -0
- package/docs/skills_parameter_schema.md +417 -0
- package/docs/swaig-reference.md +1986 -0
- package/docs/swml_service_guide.md +573 -0
- package/docs/third_party_skills.md +410 -0
- package/docs/web_service.md +666 -0
- package/examples/README.md +202 -0
- package/examples/advanced-datamap.ts +108 -0
- package/examples/advanced-dynamic-config.ts +70 -0
- package/examples/auto-vivified.ts +95 -0
- package/examples/call-flow.ts +65 -0
- package/examples/comprehensive-dynamic.ts +125 -0
- package/examples/contexts-steps.ts +65 -0
- package/examples/custom-path.ts +45 -0
- package/examples/datamap-tools.ts +51 -0
- package/examples/datasphere-multi-instance.ts +66 -0
- package/examples/datasphere-serverless-env.ts +87 -0
- package/examples/datasphere-webhook-env.ts +65 -0
- package/examples/datasphere.ts +56 -0
- package/examples/declarative.ts +100 -0
- package/examples/dynamic-config.ts +54 -0
- package/examples/dynamic-info-gatherer.ts +69 -0
- package/examples/dynamic-swml-service.ts +67 -0
- package/examples/gather-info.ts +95 -0
- package/examples/gather-per-question-functions-demo.ts +139 -0
- package/examples/joke-agent.ts +34 -0
- package/examples/kubernetes-agent.ts +45 -0
- package/examples/llm-params.ts +75 -0
- package/examples/mcp-agent.ts +89 -0
- package/examples/mcp-gateway.ts +81 -0
- package/examples/multi-agent.ts +80 -0
- package/examples/multi-endpoint.ts +79 -0
- package/examples/pgi/case-domain.ts +114 -0
- package/examples/pgi/serve.ts +16 -0
- package/examples/pgi/support-agent.ts +265 -0
- package/examples/pom-prompt.ts +70 -0
- package/examples/prefab-concierge.ts +42 -0
- package/examples/prefab-faq.ts +58 -0
- package/examples/prefab-info-gatherer.ts +33 -0
- package/examples/prefab-receptionist.ts +36 -0
- package/examples/prefab-survey.ts +69 -0
- package/examples/quickstart-agent.ts +29 -0
- package/examples/quickstart-relay.ts +25 -0
- package/examples/quickstart-rest.ts +25 -0
- package/examples/record-call.ts +65 -0
- package/examples/relay-demo.ts +33 -0
- package/examples/room-and-sip.ts +93 -0
- package/examples/serverless-lambda.ts +43 -0
- package/examples/session-state.ts +83 -0
- package/examples/simple-agent.ts +36 -0
- package/examples/simple-static.ts +29 -0
- package/examples/skills-demo.ts +41 -0
- package/examples/step-function-inheritance-demo.ts +124 -0
- package/examples/swaig-features.ts +193 -0
- package/examples/swml-service-routing.ts +66 -0
- package/examples/swml-service.ts +38 -0
- package/examples/swmlservice_ai_sidecar.ts +104 -0
- package/examples/swmlservice_swaig_standalone.ts +79 -0
- package/examples/tap.ts +67 -0
- package/examples/typed-tools.ts +68 -0
- package/examples/verb-methods.ts +45 -0
- package/examples/web-search-multi-instance.ts +75 -0
- package/examples/web-search.ts +38 -0
- package/examples/wikipedia.ts +36 -0
- package/livewire/README.md +164 -0
- package/livewire/docs/migration-guide.md +251 -0
- package/livewire/examples/livewire-basic-agent.ts +74 -0
- package/livewire/examples/livewire-handoff.ts +196 -0
- package/livewire/examples/livewire-multi-tool.ts +121 -0
- package/package.json +85 -13
- package/relay/README.md +101 -0
- package/relay/docs/call-methods.md +622 -0
- package/relay/docs/client-reference.md +281 -0
- package/relay/docs/events.md +193 -0
- package/relay/docs/getting-started.md +221 -0
- package/relay/docs/guide.md +386 -0
- package/relay/docs/messaging.md +246 -0
- package/relay/examples/relay-inbound.ts +66 -0
- package/relay/examples/relay-messaging.ts +74 -0
- package/relay/examples/relay-outbound.ts +76 -0
- package/rest/README.md +141 -0
- package/rest/docs/calling.md +313 -0
- package/rest/docs/client-reference.md +149 -0
- package/rest/docs/fabric.md +192 -0
- package/rest/docs/getting-started.md +138 -0
- package/rest/docs/guide.md +421 -0
- package/rest/docs/namespaces.md +367 -0
- package/rest/docs/phone-binding.md +116 -0
- package/rest/examples/rest-10dlc-registration.ts +177 -0
- package/rest/examples/rest-bind-phone-to-swml-webhook.ts +66 -0
- package/rest/examples/rest-calling-ivr-and-ai.ts +136 -0
- package/rest/examples/rest-calling-play-and-record.ts +98 -0
- package/rest/examples/rest-client.ts +59 -0
- package/rest/examples/rest-datasphere-search.ts +81 -0
- package/rest/examples/rest-fabric-conferences-and-routing.ts +147 -0
- package/rest/examples/rest-fabric-subscribers-and-sip.ts +106 -0
- package/rest/examples/rest-fabric-swml-and-callflows.ts +101 -0
- package/rest/examples/rest-manage-resources.ts +64 -0
- package/rest/examples/rest-phone-number-management.ts +200 -0
- package/rest/examples/rest-queues-mfa-and-recordings.ts +143 -0
- package/rest/examples/rest-video-rooms.ts +186 -0
- package/tutorial/fred/fred.ts +218 -0
- package/tutorial/fred/tutorial/01-introduction.md +141 -0
- package/tutorial/fred/tutorial/02-setup.md +304 -0
- package/tutorial/fred/tutorial/03-basic-agent.md +284 -0
- package/tutorial/fred/tutorial/04-wikipedia-skill.md +298 -0
- package/tutorial/fred/tutorial/05-custom-functions.md +347 -0
- package/tutorial/fred/tutorial/06-running-testing.md +642 -0
- package/tutorial/fred/tutorial/README.md +56 -0
- package/tutorial/fred/tutorial/appendix-complete-code.md +632 -0
- package/tutorial/fred/tutorial/appendix-docker-deployment.md +504 -0
- package/tutorial/full-guardrails-agent/handlers.ts +436 -0
- package/tutorial/full-guardrails-agent/penny.ts +305 -0
- package/tutorial/full-guardrails-agent/reservations.ts +1109 -0
- package/tutorial/full-guardrails-agent/tutorial/01-why-guardrails.md +149 -0
- package/tutorial/full-guardrails-agent/tutorial/02-design-first.md +139 -0
- package/tutorial/full-guardrails-agent/tutorial/03-rules-first.md +422 -0
- package/tutorial/full-guardrails-agent/tutorial/04-the-shell.md +238 -0
- package/tutorial/full-guardrails-agent/tutorial/05-steps-and-scoping.md +247 -0
- package/tutorial/full-guardrails-agent/tutorial/06-tools-that-decide.md +467 -0
- package/tutorial/full-guardrails-agent/tutorial/07-gather-and-projection.md +160 -0
- package/tutorial/full-guardrails-agent/tutorial/08-the-verification-gate.md +333 -0
- package/tutorial/full-guardrails-agent/tutorial/09-people-and-endings.md +313 -0
- package/tutorial/full-guardrails-agent/tutorial/10-testing-and-running.md +387 -0
- package/tutorial/full-guardrails-agent/tutorial/README.md +106 -0
- package/tutorial/full-guardrails-agent/tutorial/appendix-complete-code.md +2207 -0
- package/tutorial/full-guardrails-agent/tutorial/appendix-deployment.md +191 -0
- package/tutorial/full-guardrails-agent/tutorial/appendix-technique-map.md +117 -0
- package/tutorial/full-guardrails-agent/workflow.ts +219 -0
- package/tutorial/multi_agents/README.md +175 -0
- package/tutorial/multi_agents/advanced_agent.ts +395 -0
- package/tutorial/multi_agents/extending_agents.ts +641 -0
- package/tutorial/multi_agents/knowledge.ts +59 -0
- package/tutorial/multi_agents/lesson1_first_agent.md +368 -0
- package/tutorial/multi_agents/lesson2_knowledge_bases.md +404 -0
- package/tutorial/multi_agents/lesson3_multi_agent_systems.md +621 -0
- package/tutorial/multi_agents/lesson4_advanced_features.md +805 -0
- package/tutorial/multi_agents/lesson5_extending_agents.md +856 -0
- package/tutorial/multi_agents/pc_builder.ts +529 -0
- package/tutorial/multi_agents/sales_agent.ts +88 -0
- package/tutorial/multi_agents/sales_agent_with_search.ts +116 -0
- package/tutorial/multi_agents/sales_knowledge.md +139 -0
- package/tutorial/multi_agents/support_knowledge.md +256 -0
- package/dist/cli/mock-data.d.ts +0 -37
- package/dist/cli/mock-data.js +0 -80
- package/dist/cli/mock-data.js.map +0 -1
- package/dist/generateVerbTypes.d.ts +0 -11
- package/dist/generateVerbTypes.js +0 -265
- package/dist/generateVerbTypes.js.map +0 -1
- package/dist/rest/namespaces/compat.d.ts +0 -585
- package/dist/rest/namespaces/compat.js +0 -735
- package/dist/rest/namespaces/compat.js.map +0 -1
|
@@ -11,23 +11,16 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Multiple instances are supported; use `tool_name` in config to differentiate.
|
|
13
13
|
*/
|
|
14
|
-
import { SkillBase } from '../SkillBase.js';
|
|
14
|
+
import { SkillBase, defineSkillTool } from '../SkillBase.js';
|
|
15
15
|
import { FunctionResult } from '../../FunctionResult.js';
|
|
16
|
+
import { _publicFetch, _RedirectRefused } from '../../PublicFetch.js';
|
|
17
|
+
import { _RobotsRules } from '../../RobotsTxt.js';
|
|
16
18
|
import { resolveAndValidateUrl, validateUrl, MAX_SKILL_INPUT_LENGTH } from '../../SecurityUtils.js';
|
|
17
19
|
import { getLogger } from '../../Logger.js';
|
|
18
|
-
import * as cheerio from 'cheerio';
|
|
19
20
|
const log = getLogger('SpiderSkill');
|
|
20
21
|
const WHITESPACE_REGEX = /\s+/g;
|
|
21
|
-
/**
|
|
22
|
-
const
|
|
23
|
-
/<script\b[^>]*>[\s\S]*?<\/script>/gi,
|
|
24
|
-
/<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
|
25
|
-
/<nav\b[^>]*>[\s\S]*?<\/nav>/gi,
|
|
26
|
-
/<header\b[^>]*>[\s\S]*?<\/header>/gi,
|
|
27
|
-
/<footer\b[^>]*>[\s\S]*?<\/footer>/gi,
|
|
28
|
-
/<aside\b[^>]*>[\s\S]*?<\/aside>/gi,
|
|
29
|
-
/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi,
|
|
30
|
-
];
|
|
22
|
+
/** Headers that carry credentials for one origin (as _publicFetch treats them). */
|
|
23
|
+
const ORIGIN_BOUND_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']);
|
|
31
24
|
/**
|
|
32
25
|
* Fast web scraping skill optimized for speed and token efficiency.
|
|
33
26
|
*
|
|
@@ -39,7 +32,9 @@ const STRIP_TAG_REGEXES = [
|
|
|
39
32
|
*
|
|
40
33
|
* @example
|
|
41
34
|
* ```ts
|
|
42
|
-
*
|
|
35
|
+
* import { AgentBase } from '@signalwire/sdk';
|
|
36
|
+
* const agent = new AgentBase({ name: 'demo', route: '/' });
|
|
37
|
+
* await agent.addSkillByName('spider', { max_pages: 5, max_depth: 2 });
|
|
43
38
|
* ```
|
|
44
39
|
*/
|
|
45
40
|
export class SpiderSkill extends SkillBase {
|
|
@@ -51,20 +46,64 @@ export class SpiderSkill extends SkillBase {
|
|
|
51
46
|
static REQUIRED_PACKAGES = ['cheerio'];
|
|
52
47
|
static REQUIRED_ENV_VARS = [];
|
|
53
48
|
static SUPPORTS_MULTIPLE_INSTANCES = true;
|
|
49
|
+
/**
|
|
50
|
+
* Each setting's default. The parameter schema and setup() both read this,
|
|
51
|
+
* so the schema can't advertise a default the skill doesn't use.
|
|
52
|
+
*/
|
|
53
|
+
static DEFAULTS = {
|
|
54
|
+
delay: 0.1,
|
|
55
|
+
concurrent_requests: 5,
|
|
56
|
+
timeout: 5,
|
|
57
|
+
max_pages: 1,
|
|
58
|
+
max_depth: 0,
|
|
59
|
+
extract_type: 'fast_text',
|
|
60
|
+
max_text_length: 3000,
|
|
61
|
+
clean_text: true,
|
|
62
|
+
cache_enabled: true,
|
|
63
|
+
follow_robots_txt: false,
|
|
64
|
+
user_agent: 'Spider/1.0 (SignalWire AI Agent)',
|
|
65
|
+
};
|
|
66
|
+
/** The extraction methods scrape_url implements. */
|
|
67
|
+
static EXTRACT_TYPES = [
|
|
68
|
+
'fast_text',
|
|
69
|
+
'markdown',
|
|
70
|
+
'structured',
|
|
71
|
+
];
|
|
72
|
+
/**
|
|
73
|
+
* Values the schema once listed that were never implemented. They have
|
|
74
|
+
* always worked as fast_text, so they still do, with a warning.
|
|
75
|
+
*/
|
|
76
|
+
static LEGACY_EXTRACT_TYPES = new Set([
|
|
77
|
+
'clean_text',
|
|
78
|
+
'full_text',
|
|
79
|
+
'html',
|
|
80
|
+
'custom',
|
|
81
|
+
]);
|
|
82
|
+
/** Seconds to keep a site's robots.txt rules; RFC 9309 allows up to 24 hours. */
|
|
83
|
+
static ROBOTS_TTL = 24 * 60 * 60;
|
|
54
84
|
static getParameterSchema() {
|
|
85
|
+
const d = SpiderSkill.DEFAULTS;
|
|
55
86
|
return {
|
|
56
87
|
...super.getParameterSchema(),
|
|
88
|
+
// The base schema's default is the skill name, but tool_name is a
|
|
89
|
+
// prefix that's empty when unset (Python skill.py:306 too).
|
|
90
|
+
tool_name: {
|
|
91
|
+
type: 'string',
|
|
92
|
+
description: 'Prefix for the tool names, as in <tool_name>_scrape_url, and for the instance key. ' +
|
|
93
|
+
'Unset, the tools keep their plain names.',
|
|
94
|
+
required: false,
|
|
95
|
+
},
|
|
57
96
|
delay: {
|
|
58
97
|
type: 'number',
|
|
59
98
|
description: 'Delay between requests in seconds',
|
|
60
|
-
default:
|
|
99
|
+
default: d.delay,
|
|
61
100
|
required: false,
|
|
62
101
|
min: 0,
|
|
63
102
|
},
|
|
64
103
|
concurrent_requests: {
|
|
65
104
|
type: 'integer',
|
|
66
|
-
description: '
|
|
67
|
-
default:
|
|
105
|
+
description: 'Deprecated, and has no effect: the spider fetches one page at a time',
|
|
106
|
+
default: d.concurrent_requests,
|
|
68
107
|
required: false,
|
|
69
108
|
min: 1,
|
|
70
109
|
max: 20,
|
|
@@ -72,7 +111,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
72
111
|
timeout: {
|
|
73
112
|
type: 'integer',
|
|
74
113
|
description: 'Request timeout in seconds',
|
|
75
|
-
default:
|
|
114
|
+
default: d.timeout,
|
|
76
115
|
required: false,
|
|
77
116
|
min: 1,
|
|
78
117
|
max: 60,
|
|
@@ -80,7 +119,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
80
119
|
max_pages: {
|
|
81
120
|
type: 'integer',
|
|
82
121
|
description: 'Maximum number of pages to scrape',
|
|
83
|
-
default:
|
|
122
|
+
default: d.max_pages,
|
|
84
123
|
required: false,
|
|
85
124
|
min: 1,
|
|
86
125
|
max: 100,
|
|
@@ -88,7 +127,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
88
127
|
max_depth: {
|
|
89
128
|
type: 'integer',
|
|
90
129
|
description: 'Maximum crawl depth (0 = single page only)',
|
|
91
|
-
default:
|
|
130
|
+
default: d.max_depth,
|
|
92
131
|
required: false,
|
|
93
132
|
min: 0,
|
|
94
133
|
max: 5,
|
|
@@ -96,14 +135,14 @@ export class SpiderSkill extends SkillBase {
|
|
|
96
135
|
extract_type: {
|
|
97
136
|
type: 'string',
|
|
98
137
|
description: 'Content extraction method',
|
|
99
|
-
default:
|
|
138
|
+
default: d.extract_type,
|
|
100
139
|
required: false,
|
|
101
|
-
enum: [
|
|
140
|
+
enum: [...SpiderSkill.EXTRACT_TYPES],
|
|
102
141
|
},
|
|
103
142
|
max_text_length: {
|
|
104
143
|
type: 'integer',
|
|
105
144
|
description: 'Maximum text length to return',
|
|
106
|
-
default:
|
|
145
|
+
default: d.max_text_length,
|
|
107
146
|
required: false,
|
|
108
147
|
min: 100,
|
|
109
148
|
max: 100000,
|
|
@@ -111,7 +150,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
111
150
|
clean_text: {
|
|
112
151
|
type: 'boolean',
|
|
113
152
|
description: 'Whether to clean extracted text',
|
|
114
|
-
default:
|
|
153
|
+
default: d.clean_text,
|
|
115
154
|
required: false,
|
|
116
155
|
},
|
|
117
156
|
selectors: {
|
|
@@ -131,7 +170,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
131
170
|
user_agent: {
|
|
132
171
|
type: 'string',
|
|
133
172
|
description: 'User agent string for requests',
|
|
134
|
-
default:
|
|
173
|
+
default: d.user_agent,
|
|
135
174
|
required: false,
|
|
136
175
|
},
|
|
137
176
|
headers: {
|
|
@@ -143,63 +182,72 @@ export class SpiderSkill extends SkillBase {
|
|
|
143
182
|
},
|
|
144
183
|
follow_robots_txt: {
|
|
145
184
|
type: 'boolean',
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
// True but the runtime and TS both use False — aligning the schema
|
|
149
|
-
// eliminates the cross-SDK contract mismatch.
|
|
150
|
-
description: 'Whether to respect robots.txt (default: false to match Python runtime)',
|
|
151
|
-
default: false,
|
|
185
|
+
description: "Skip pages that the site's robots.txt disallows for user_agent",
|
|
186
|
+
default: d.follow_robots_txt,
|
|
152
187
|
required: false,
|
|
153
188
|
},
|
|
154
189
|
cache_enabled: {
|
|
155
190
|
type: 'boolean',
|
|
156
191
|
description: 'Whether to cache scraped pages',
|
|
157
|
-
default:
|
|
192
|
+
default: d.cache_enabled,
|
|
158
193
|
required: false,
|
|
159
194
|
},
|
|
160
195
|
};
|
|
161
196
|
}
|
|
162
197
|
// Runtime state, populated in setup()
|
|
163
|
-
delay =
|
|
164
|
-
concurrentRequests =
|
|
165
|
-
timeout =
|
|
166
|
-
maxPages =
|
|
167
|
-
maxDepth =
|
|
168
|
-
extractType =
|
|
169
|
-
maxTextLength =
|
|
170
|
-
cleanText =
|
|
171
|
-
cacheEnabled =
|
|
172
|
-
followRobotsTxt =
|
|
173
|
-
userAgent =
|
|
198
|
+
delay = SpiderSkill.DEFAULTS.delay;
|
|
199
|
+
concurrentRequests = SpiderSkill.DEFAULTS.concurrent_requests;
|
|
200
|
+
timeout = SpiderSkill.DEFAULTS.timeout;
|
|
201
|
+
maxPages = SpiderSkill.DEFAULTS.max_pages;
|
|
202
|
+
maxDepth = SpiderSkill.DEFAULTS.max_depth;
|
|
203
|
+
extractType = SpiderSkill.DEFAULTS.extract_type;
|
|
204
|
+
maxTextLength = SpiderSkill.DEFAULTS.max_text_length;
|
|
205
|
+
cleanText = SpiderSkill.DEFAULTS.clean_text;
|
|
206
|
+
cacheEnabled = SpiderSkill.DEFAULTS.cache_enabled;
|
|
207
|
+
followRobotsTxt = SpiderSkill.DEFAULTS.follow_robots_txt;
|
|
208
|
+
userAgent = SpiderSkill.DEFAULTS.user_agent;
|
|
174
209
|
headers = {};
|
|
175
210
|
selectors = {};
|
|
176
211
|
compiledFollowPatterns = [];
|
|
177
212
|
cache = null;
|
|
178
213
|
cacheMaxSize = 100;
|
|
214
|
+
/** robots.txt rules per origin, with when they expire, when follow_robots_txt is on. */
|
|
215
|
+
robots = new Map();
|
|
216
|
+
// Lazily-loaded optional `cheerio` module, populated in setup(). Non-null
|
|
217
|
+
// for the lifetime of an initialized skill (setup() returns false if absent).
|
|
218
|
+
_cheerio;
|
|
179
219
|
getInstanceKey() {
|
|
180
220
|
const toolName = this.getConfig('tool_name', this.skillName);
|
|
181
221
|
return `${this.skillName}_${toolName}`;
|
|
182
222
|
}
|
|
183
223
|
async setup() {
|
|
224
|
+
// Load the optional cheerio dependency once. If it isn't installed, the
|
|
225
|
+
// skill cannot scrape — fail setup cleanly (validatePackages() also flags
|
|
226
|
+
// it). Cached on the instance so the sync extract helpers can use it.
|
|
227
|
+
try {
|
|
228
|
+
this._cheerio = await import('cheerio');
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
log.error('spider: the optional "cheerio" package is required for this skill but is not installed');
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
const d = SpiderSkill.DEFAULTS;
|
|
184
235
|
// Performance
|
|
185
|
-
this.delay = this.getConfig('delay',
|
|
186
|
-
this.concurrentRequests = this.getConfig('concurrent_requests',
|
|
187
|
-
this.timeout = this.getConfig('timeout',
|
|
236
|
+
this.delay = this.getConfig('delay', d.delay);
|
|
237
|
+
this.concurrentRequests = this.getConfig('concurrent_requests', d.concurrent_requests);
|
|
238
|
+
this.timeout = this.getConfig('timeout', d.timeout);
|
|
188
239
|
// Crawl limits
|
|
189
|
-
this.maxPages = this.getConfig('max_pages',
|
|
190
|
-
this.maxDepth = this.getConfig('max_depth',
|
|
240
|
+
this.maxPages = this.getConfig('max_pages', d.max_pages);
|
|
241
|
+
this.maxDepth = this.getConfig('max_depth', d.max_depth);
|
|
191
242
|
// Content processing
|
|
192
|
-
this.extractType = this.getConfig('extract_type',
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
// `self.params.get('max_text_length', 3000)` and schema defaults are not
|
|
196
|
-
// applied. Use 3000 for parity. Follow-up: reconcile Python schema vs init.
|
|
197
|
-
this.maxTextLength = this.getConfig('max_text_length', 3000);
|
|
198
|
-
this.cleanText = this.getConfig('clean_text', true);
|
|
243
|
+
this.extractType = this.getConfig('extract_type', d.extract_type);
|
|
244
|
+
this.maxTextLength = this.getConfig('max_text_length', d.max_text_length);
|
|
245
|
+
this.cleanText = this.getConfig('clean_text', d.clean_text);
|
|
199
246
|
// Features
|
|
200
|
-
this.cacheEnabled = this.getConfig('cache_enabled',
|
|
201
|
-
this.followRobotsTxt = this.getConfig('follow_robots_txt',
|
|
202
|
-
this.userAgent = this.getConfig('user_agent',
|
|
247
|
+
this.cacheEnabled = this.getConfig('cache_enabled', d.cache_enabled);
|
|
248
|
+
this.followRobotsTxt = this.getConfig('follow_robots_txt', d.follow_robots_txt);
|
|
249
|
+
this.userAgent = this.getConfig('user_agent', d.user_agent);
|
|
250
|
+
this.robots = new Map();
|
|
203
251
|
// Optional headers + user-agent merge
|
|
204
252
|
const headers = this.getConfig('headers', {}) ?? {};
|
|
205
253
|
this.headers = { ...headers, 'User-Agent': this.userAgent };
|
|
@@ -213,7 +261,22 @@ export class SpiderSkill extends SkillBase {
|
|
|
213
261
|
return false;
|
|
214
262
|
}
|
|
215
263
|
if (this.concurrentRequests < 1 || this.concurrentRequests > 20) {
|
|
216
|
-
log.error('spider: concurrent_requests must be between 1 and 20', {
|
|
264
|
+
log.error('spider: concurrent_requests must be between 1 and 20', {
|
|
265
|
+
concurrent_requests: this.concurrentRequests,
|
|
266
|
+
});
|
|
267
|
+
return false;
|
|
268
|
+
}
|
|
269
|
+
if ('concurrent_requests' in this.config) {
|
|
270
|
+
log.warn('spider: concurrent_requests is deprecated and has no effect: the spider fetches one page at a time');
|
|
271
|
+
}
|
|
272
|
+
// Validate the extraction method
|
|
273
|
+
const types = SpiderSkill.EXTRACT_TYPES.join(', ');
|
|
274
|
+
if (SpiderSkill.LEGACY_EXTRACT_TYPES.has(this.extractType)) {
|
|
275
|
+
log.warn(`spider: extract_type '${this.extractType}' was never implemented and works as fast_text; use one of ${types}`);
|
|
276
|
+
this.extractType = 'fast_text';
|
|
277
|
+
}
|
|
278
|
+
else if (!SpiderSkill.EXTRACT_TYPES.includes(this.extractType)) {
|
|
279
|
+
log.error(`spider: unknown extract_type '${this.extractType}'; use one of ${types}`);
|
|
217
280
|
return false;
|
|
218
281
|
}
|
|
219
282
|
if (this.maxPages < 1) {
|
|
@@ -267,8 +330,11 @@ export class SpiderSkill extends SkillBase {
|
|
|
267
330
|
getTools() {
|
|
268
331
|
const toolPrefix = this.getConfig('tool_name', '');
|
|
269
332
|
const prefix = toolPrefix ? `${toolPrefix}_` : '';
|
|
333
|
+
// The three handlers keep their defensive typeof/.trim()/URL-validation
|
|
334
|
+
// narrowing in the private methods: `url`/`start_url` are required, but the
|
|
335
|
+
// model can still emit an empty or wrong-typed value.
|
|
270
336
|
return [
|
|
271
|
-
{
|
|
337
|
+
defineSkillTool({
|
|
272
338
|
name: `${prefix}scrape_url`,
|
|
273
339
|
description: 'Extract text content from a single web page',
|
|
274
340
|
parameters: {
|
|
@@ -279,8 +345,8 @@ export class SpiderSkill extends SkillBase {
|
|
|
279
345
|
},
|
|
280
346
|
required: ['url'],
|
|
281
347
|
handler: async (args, rawData) => this._scrapeUrlHandler(args, rawData),
|
|
282
|
-
},
|
|
283
|
-
{
|
|
348
|
+
}),
|
|
349
|
+
defineSkillTool({
|
|
284
350
|
name: `${prefix}crawl_site`,
|
|
285
351
|
description: 'Crawl multiple pages starting from a URL',
|
|
286
352
|
parameters: {
|
|
@@ -291,8 +357,8 @@ export class SpiderSkill extends SkillBase {
|
|
|
291
357
|
},
|
|
292
358
|
required: ['start_url'],
|
|
293
359
|
handler: async (args, rawData) => this._crawlSiteHandler(args, rawData),
|
|
294
|
-
},
|
|
295
|
-
{
|
|
360
|
+
}),
|
|
361
|
+
defineSkillTool({
|
|
296
362
|
name: `${prefix}extract_structured_data`,
|
|
297
363
|
description: 'Extract specific data from a web page using selectors',
|
|
298
364
|
parameters: {
|
|
@@ -303,18 +369,30 @@ export class SpiderSkill extends SkillBase {
|
|
|
303
369
|
},
|
|
304
370
|
required: ['url'],
|
|
305
371
|
handler: async (args, rawData) => this._extractStructuredHandler(args, rawData),
|
|
306
|
-
},
|
|
372
|
+
}),
|
|
307
373
|
];
|
|
308
374
|
}
|
|
309
375
|
// ---------------------------------------------------------------------------
|
|
310
376
|
// Internal helpers
|
|
311
377
|
// ---------------------------------------------------------------------------
|
|
312
378
|
/**
|
|
313
|
-
* Redirect a target URL through `SPIDER_BASE_URL` when set. Used by
|
|
314
|
-
*
|
|
315
|
-
*
|
|
379
|
+
* Redirect a target URL through `SPIDER_BASE_URL` when set. Used by tests
|
|
380
|
+
* to point every outbound fetch at a loopback fixture without requiring
|
|
381
|
+
* the test URL to resolve.
|
|
316
382
|
* Preserves the path (and any query/fragment) from the original target.
|
|
317
383
|
*/
|
|
384
|
+
/** Whether a URL is on the operator-set SPIDER_BASE_URL's origin, a trusted target. */
|
|
385
|
+
static _isAuditOrigin(target) {
|
|
386
|
+
const base = process.env['SPIDER_BASE_URL'];
|
|
387
|
+
if (!base)
|
|
388
|
+
return false;
|
|
389
|
+
try {
|
|
390
|
+
return new URL(target).origin === new URL(base).origin;
|
|
391
|
+
}
|
|
392
|
+
catch {
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
318
396
|
static _redirectForAudit(target) {
|
|
319
397
|
const base = process.env['SPIDER_BASE_URL'];
|
|
320
398
|
if (!base)
|
|
@@ -331,6 +409,94 @@ export class SpiderSkill extends SkillBase {
|
|
|
331
409
|
return `${base.replace(/\/+$/, '')}${path}`;
|
|
332
410
|
}
|
|
333
411
|
}
|
|
412
|
+
/**
|
|
413
|
+
* Whether a URL the caller supplied may be fetched: it must not be private
|
|
414
|
+
* or internal (`SWML_ALLOW_PRIVATE_URLS` allows it). With `SPIDER_BASE_URL`
|
|
415
|
+
* set, the fetch goes to that operator-configured base instead of the URL's
|
|
416
|
+
* host, so the host isn't checked.
|
|
417
|
+
*/
|
|
418
|
+
static async _targetAllowed(url) {
|
|
419
|
+
if (SpiderSkill._redirectForAudit(url) !== url)
|
|
420
|
+
return true;
|
|
421
|
+
try {
|
|
422
|
+
await resolveAndValidateUrl(url);
|
|
423
|
+
return true;
|
|
424
|
+
}
|
|
425
|
+
catch {
|
|
426
|
+
return false;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* False when follow_robots_txt is on and the site's robots.txt disallows
|
|
431
|
+
* `url` for user_agent.
|
|
432
|
+
*
|
|
433
|
+
* As in Python's `urllib.robotparser`, a robots.txt answered with 401 or
|
|
434
|
+
* 403 disallows everything and any other 4xx allows everything. Rules are
|
|
435
|
+
* kept for {@link ROBOTS_TTL}. A server error or failed request disallows
|
|
436
|
+
* this page without keeping anything, so the next request tries again.
|
|
437
|
+
*/
|
|
438
|
+
async _allowedByRobots(url, opts = {}) {
|
|
439
|
+
if (!this.followRobotsTxt)
|
|
440
|
+
return true;
|
|
441
|
+
let origin;
|
|
442
|
+
try {
|
|
443
|
+
// Check the URL as it will be requested: fetch normalizes it, so
|
|
444
|
+
// /public/../private is a request for /private.
|
|
445
|
+
const parsed = new URL(url);
|
|
446
|
+
url = parsed.href;
|
|
447
|
+
origin = parsed.origin;
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
452
|
+
const now = performance.now() / 1000;
|
|
453
|
+
const cached = this.robots.get(origin);
|
|
454
|
+
if (cached && cached.expires > now)
|
|
455
|
+
return cached.rules.canFetch(this.userAgent, url);
|
|
456
|
+
const robotsUrl = `${origin}/robots.txt`;
|
|
457
|
+
const fetchUrl = SpiderSkill._redirectForAudit(robotsUrl);
|
|
458
|
+
// Credentials for the page's origin don't go to another origin's
|
|
459
|
+
// robots.txt, as _publicFetch drops them on a redirect there.
|
|
460
|
+
const headers = { ...this.headers };
|
|
461
|
+
if (opts.withoutCredentials) {
|
|
462
|
+
for (const name of Object.keys(headers)) {
|
|
463
|
+
if (ORIGIN_BOUND_HEADERS.has(name.toLowerCase()))
|
|
464
|
+
delete headers[name];
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
let status;
|
|
468
|
+
let body = '';
|
|
469
|
+
const controller = new AbortController();
|
|
470
|
+
const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
|
|
471
|
+
try {
|
|
472
|
+
const response = await _publicFetch(fetchUrl, {
|
|
473
|
+
method: 'GET',
|
|
474
|
+
headers,
|
|
475
|
+
signal: controller.signal,
|
|
476
|
+
allowPrivate: fetchUrl !== robotsUrl || SpiderSkill._isAuditOrigin(robotsUrl),
|
|
477
|
+
});
|
|
478
|
+
status = response.status;
|
|
479
|
+
body = await response.text();
|
|
480
|
+
}
|
|
481
|
+
catch {
|
|
482
|
+
status = 599;
|
|
483
|
+
}
|
|
484
|
+
finally {
|
|
485
|
+
clearTimeout(timer);
|
|
486
|
+
}
|
|
487
|
+
// Unavailable for now: disallow this request, and try again next time.
|
|
488
|
+
if (status >= 500)
|
|
489
|
+
return false;
|
|
490
|
+
let rules;
|
|
491
|
+
if (status === 401 || status === 403)
|
|
492
|
+
rules = _RobotsRules.disallowAll();
|
|
493
|
+
else if (status >= 400)
|
|
494
|
+
rules = _RobotsRules.allowAll();
|
|
495
|
+
else
|
|
496
|
+
rules = new _RobotsRules(body);
|
|
497
|
+
this.robots.set(origin, { rules, expires: now + SpiderSkill.ROBOTS_TTL });
|
|
498
|
+
return rules.canFetch(this.userAgent, url);
|
|
499
|
+
}
|
|
334
500
|
/** Fetch a URL with caching and timeout handling. Returns null on failure. */
|
|
335
501
|
async _fetchUrl(url) {
|
|
336
502
|
if (this.cacheEnabled && this.cache?.has(url)) {
|
|
@@ -342,13 +508,29 @@ export class SpiderSkill extends SkillBase {
|
|
|
342
508
|
// a `https://audit.example/page` target and expects the skill to hit
|
|
343
509
|
// `http://127.0.0.1:NNNN/page` — preserving the path after the host.
|
|
344
510
|
const fetchUrl = SpiderSkill._redirectForAudit(url);
|
|
511
|
+
const startOrigin = new URL(fetchUrl).origin;
|
|
512
|
+
let crossedOrigin = false;
|
|
345
513
|
const controller = new AbortController();
|
|
346
514
|
const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
|
|
347
515
|
try {
|
|
348
|
-
|
|
516
|
+
// _publicFetch checks the URL and every redirect, and refuses a
|
|
517
|
+
// connection to a private or internal address. An operator-set
|
|
518
|
+
// SPIDER_BASE_URL is a trusted target, so its fetches skip the check.
|
|
519
|
+
const response = await _publicFetch(fetchUrl, {
|
|
349
520
|
method: 'GET',
|
|
350
521
|
headers: this.headers,
|
|
351
522
|
signal: controller.signal,
|
|
523
|
+
allowPrivate: fetchUrl !== url,
|
|
524
|
+
// Each redirect's target needs its own robots.txt check.
|
|
525
|
+
allowRedirect: this.followRobotsTxt
|
|
526
|
+
? (target) => {
|
|
527
|
+
// Once a redirect leaves the page's origin, _publicFetch has
|
|
528
|
+
// dropped the credentials for good, and so does robots.txt.
|
|
529
|
+
if (new URL(target).origin !== startOrigin)
|
|
530
|
+
crossedOrigin = true;
|
|
531
|
+
return this._allowedByRobots(target, { withoutCredentials: crossedOrigin });
|
|
532
|
+
}
|
|
533
|
+
: undefined,
|
|
352
534
|
});
|
|
353
535
|
if (!response.ok) {
|
|
354
536
|
log.error('spider: fetch failed', { url, status: response.status });
|
|
@@ -371,6 +553,10 @@ export class SpiderSkill extends SkillBase {
|
|
|
371
553
|
return cached;
|
|
372
554
|
}
|
|
373
555
|
catch (err) {
|
|
556
|
+
if (err instanceof _RedirectRefused) {
|
|
557
|
+
log.info('spider: robots.txt disallows the redirect', { url, target: err.target });
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
374
560
|
log.error('spider: fetch error', {
|
|
375
561
|
url,
|
|
376
562
|
error: err instanceof Error ? err.message : String(err),
|
|
@@ -411,18 +597,10 @@ export class SpiderSkill extends SkillBase {
|
|
|
411
597
|
// decoded (Python's lxml does this natively). Previously TS only
|
|
412
598
|
// decoded six hand-coded entities, leaving `—`, `—`, etc.
|
|
413
599
|
// literal in the output.
|
|
414
|
-
const $ =
|
|
600
|
+
const $ = this._cheerio.load(body);
|
|
415
601
|
// Strip noise tags before extracting text — matches Python's
|
|
416
602
|
// lxml drop_tree() on the same 7 tags.
|
|
417
|
-
for (const tag of [
|
|
418
|
-
'script',
|
|
419
|
-
'style',
|
|
420
|
-
'nav',
|
|
421
|
-
'header',
|
|
422
|
-
'footer',
|
|
423
|
-
'aside',
|
|
424
|
-
'noscript',
|
|
425
|
-
]) {
|
|
603
|
+
for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside', 'noscript']) {
|
|
426
604
|
$(tag).remove();
|
|
427
605
|
}
|
|
428
606
|
let text = $('body').text();
|
|
@@ -438,9 +616,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
438
616
|
const keepStart = Math.floor((this.maxTextLength * 2) / 3);
|
|
439
617
|
const keepEnd = Math.floor(this.maxTextLength / 3);
|
|
440
618
|
text =
|
|
441
|
-
text.slice(0, keepStart) +
|
|
442
|
-
'\n\n[...CONTENT TRUNCATED...]\n\n' +
|
|
443
|
-
text.slice(-keepEnd);
|
|
619
|
+
text.slice(0, keepStart) + '\n\n[...CONTENT TRUNCATED...]\n\n' + text.slice(-keepEnd);
|
|
444
620
|
}
|
|
445
621
|
return text;
|
|
446
622
|
}
|
|
@@ -483,7 +659,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
483
659
|
/** Markdown extraction using cheerio. Mirrors Python `_markdown_extract`. */
|
|
484
660
|
_markdownExtract(response) {
|
|
485
661
|
try {
|
|
486
|
-
const $ =
|
|
662
|
+
const $ = this._cheerio.load(response.body);
|
|
487
663
|
// Remove unwanted tags
|
|
488
664
|
for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside']) {
|
|
489
665
|
$(tag).remove();
|
|
@@ -527,7 +703,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
527
703
|
_structuredExtract(cached, selectors) {
|
|
528
704
|
let $;
|
|
529
705
|
try {
|
|
530
|
-
$ =
|
|
706
|
+
$ = this._cheerio.load(cached.body);
|
|
531
707
|
}
|
|
532
708
|
catch (err) {
|
|
533
709
|
// Python returns `{'error': str(e)}` from _structured_extract when the
|
|
@@ -585,13 +761,12 @@ export class SpiderSkill extends SkillBase {
|
|
|
585
761
|
return new FunctionResult(`Invalid URL: ${url}`);
|
|
586
762
|
}
|
|
587
763
|
// SSRF protection
|
|
588
|
-
|
|
589
|
-
try {
|
|
590
|
-
await resolveAndValidateUrl(url, allowPrivate);
|
|
591
|
-
}
|
|
592
|
-
catch {
|
|
764
|
+
if (!(await SpiderSkill._targetAllowed(url))) {
|
|
593
765
|
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
594
766
|
}
|
|
767
|
+
if (!(await this._allowedByRobots(url))) {
|
|
768
|
+
return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
|
|
769
|
+
}
|
|
595
770
|
const cached = await this._fetchUrl(url);
|
|
596
771
|
if (!cached) {
|
|
597
772
|
return new FunctionResult(`Failed to fetch ${url}`);
|
|
@@ -628,11 +803,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
628
803
|
if (!/^https?:\/\//.test(startUrl)) {
|
|
629
804
|
return new FunctionResult(`Invalid URL: ${startUrl}`);
|
|
630
805
|
}
|
|
631
|
-
|
|
632
|
-
try {
|
|
633
|
-
await resolveAndValidateUrl(startUrl, allowPrivate);
|
|
634
|
-
}
|
|
635
|
-
catch {
|
|
806
|
+
if (!(await SpiderSkill._targetAllowed(startUrl))) {
|
|
636
807
|
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
637
808
|
}
|
|
638
809
|
const maxDepth = this.maxDepth;
|
|
@@ -642,6 +813,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
642
813
|
if (maxPages < 1)
|
|
643
814
|
return new FunctionResult('Max pages must be at least 1');
|
|
644
815
|
const visited = new Set();
|
|
816
|
+
const disallowed = new Set();
|
|
645
817
|
const toVisit = [[startUrl, 0]];
|
|
646
818
|
const results = [];
|
|
647
819
|
let startHost;
|
|
@@ -656,8 +828,13 @@ export class SpiderSkill extends SkillBase {
|
|
|
656
828
|
if (!next)
|
|
657
829
|
break;
|
|
658
830
|
const [url, depth] = next;
|
|
659
|
-
if (visited.has(url) || depth > maxDepth)
|
|
831
|
+
if (visited.has(url) || disallowed.has(url) || depth > maxDepth)
|
|
832
|
+
continue;
|
|
833
|
+
if (!(await this._allowedByRobots(url))) {
|
|
834
|
+
log.info('spider: robots.txt disallows a page; skipping it', { url });
|
|
835
|
+
disallowed.add(url);
|
|
660
836
|
continue;
|
|
837
|
+
}
|
|
661
838
|
const cached = await this._fetchUrl(url);
|
|
662
839
|
if (!cached)
|
|
663
840
|
continue;
|
|
@@ -677,7 +854,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
677
854
|
// xpath('//a[@href]/@href') which has the same robustness).
|
|
678
855
|
if (depth < maxDepth) {
|
|
679
856
|
try {
|
|
680
|
-
const $links =
|
|
857
|
+
const $links = this._cheerio.load(cached.body);
|
|
681
858
|
const hrefs = [];
|
|
682
859
|
$links('a[href]').each((_, el) => {
|
|
683
860
|
const href = $links(el).attr('href');
|
|
@@ -733,10 +910,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
733
910
|
if (results.length === 0) {
|
|
734
911
|
return new FunctionResult(`No pages could be crawled from ${startUrl}`);
|
|
735
912
|
}
|
|
736
|
-
const lines = [
|
|
737
|
-
`Crawled ${results.length} pages from ${startHost}:`,
|
|
738
|
-
'',
|
|
739
|
-
];
|
|
913
|
+
const lines = [`Crawled ${results.length} pages from ${startHost}:`, ''];
|
|
740
914
|
results.forEach((r, i) => {
|
|
741
915
|
lines.push(`${i + 1}. ${r.url} (depth: ${r.depth}, ${r.contentLength} chars)`);
|
|
742
916
|
lines.push(` Summary: ${r.summary.slice(0, 100)}...`);
|
|
@@ -756,16 +930,15 @@ export class SpiderSkill extends SkillBase {
|
|
|
756
930
|
if (!/^https?:\/\//.test(url)) {
|
|
757
931
|
return new FunctionResult(`Invalid URL: ${url}`);
|
|
758
932
|
}
|
|
759
|
-
|
|
760
|
-
try {
|
|
761
|
-
await resolveAndValidateUrl(url, allowPrivate);
|
|
762
|
-
}
|
|
763
|
-
catch {
|
|
933
|
+
if (!(await SpiderSkill._targetAllowed(url))) {
|
|
764
934
|
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
765
935
|
}
|
|
766
936
|
if (!this.selectors || Object.keys(this.selectors).length === 0) {
|
|
767
937
|
return new FunctionResult('No selectors configured for structured data extraction');
|
|
768
938
|
}
|
|
939
|
+
if (!(await this._allowedByRobots(url))) {
|
|
940
|
+
return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
|
|
941
|
+
}
|
|
769
942
|
const cached = await this._fetchUrl(url);
|
|
770
943
|
if (!cached) {
|
|
771
944
|
return new FunctionResult(`Failed to fetch ${url}`);
|