@signalwire/sdk 2.0.4 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +785 -0
- package/LICENSE +21 -0
- package/README.md +163 -68
- package/dist/AGENTS.md +13 -0
- package/dist/AgentBase.d.ts +550 -163
- package/dist/AgentBase.js +1640 -514
- package/dist/AgentBase.js.map +1 -1
- package/dist/AgentServer.d.ts +21 -0
- package/dist/AgentServer.js +154 -47
- package/dist/AgentServer.js.map +1 -1
- package/dist/AuthHandler.d.ts +14 -2
- package/dist/AuthHandler.js +7 -2
- package/dist/AuthHandler.js.map +1 -1
- package/dist/ConfigLoader.d.ts +5 -13
- package/dist/ConfigLoader.js +15 -29
- package/dist/ConfigLoader.js.map +1 -1
- package/dist/ContextBuilder.d.ts +202 -9
- package/dist/ContextBuilder.js +219 -36
- package/dist/ContextBuilder.js.map +1 -1
- package/dist/DataMap.d.ts +158 -45
- package/dist/DataMap.js +167 -52
- package/dist/DataMap.js.map +1 -1
- package/dist/FunctionResult.d.ts +277 -83
- package/dist/FunctionResult.js +444 -116
- package/dist/FunctionResult.js.map +1 -1
- package/dist/Logger.d.ts +30 -5
- package/dist/Logger.js +77 -19
- package/dist/Logger.js.map +1 -1
- package/dist/POM/PromptObjectModel.d.ts +163 -0
- package/dist/POM/PromptObjectModel.js +481 -0
- package/dist/POM/PromptObjectModel.js.map +1 -0
- package/dist/ParameterSchema.d.ts +306 -0
- package/dist/ParameterSchema.js +251 -0
- package/dist/ParameterSchema.js.map +1 -0
- package/dist/PlatformContracts.d.ts +21 -0
- package/dist/PlatformContracts.generated.d.ts +109 -0
- package/dist/PlatformContracts.generated.js +8 -0
- package/dist/PlatformContracts.generated.js.map +1 -0
- package/dist/PlatformContracts.js +2 -0
- package/dist/PlatformContracts.js.map +1 -0
- package/dist/PomBuilder.d.ts +28 -0
- package/dist/PomBuilder.js +81 -3
- package/dist/PomBuilder.js.map +1 -1
- package/dist/PostPrompt.d.ts +110 -0
- package/dist/PostPrompt.js +204 -0
- package/dist/PostPrompt.js.map +1 -0
- package/dist/PromptManager.d.ts +29 -1
- package/dist/PromptManager.js +37 -1
- package/dist/PromptManager.js.map +1 -1
- package/dist/PublicFetch.d.ts +132 -0
- package/dist/PublicFetch.js +311 -0
- package/dist/PublicFetch.js.map +1 -0
- package/dist/RobotsTxt.d.ts +41 -0
- package/dist/RobotsTxt.js +152 -0
- package/dist/RobotsTxt.js.map +1 -0
- package/dist/SWMLHandler.js.map +1 -1
- package/dist/SWMLService.d.ts +265 -66
- package/dist/SWMLService.js +652 -112
- package/dist/SWMLService.js.map +1 -1
- package/dist/SchemaUtils.d.ts +115 -2
- package/dist/SchemaUtils.js +444 -12
- package/dist/SchemaUtils.js.map +1 -1
- package/dist/SecurityUtils.d.ts +55 -8
- package/dist/SecurityUtils.js +162 -49
- package/dist/SecurityUtils.js.map +1 -1
- package/dist/ServerlessAdapter.d.ts +156 -7
- package/dist/ServerlessAdapter.js +399 -42
- package/dist/ServerlessAdapter.js.map +1 -1
- package/dist/SessionManager.d.ts +11 -2
- package/dist/SessionManager.js +50 -18
- package/dist/SessionManager.js.map +1 -1
- package/dist/SslConfig.d.ts +2 -1
- package/dist/SslConfig.js +4 -1
- package/dist/SslConfig.js.map +1 -1
- package/dist/SwaigActions.generated.d.ts +23 -0
- package/dist/SwaigActions.generated.js +8 -0
- package/dist/SwaigActions.generated.js.map +1 -0
- package/dist/SwaigContracts.d.ts +15 -0
- package/dist/SwaigContracts.generated.d.ts +264 -0
- package/dist/SwaigContracts.generated.js +12 -0
- package/dist/SwaigContracts.generated.js.map +1 -0
- package/dist/SwaigContracts.js +2 -0
- package/dist/SwaigContracts.js.map +1 -0
- package/dist/SwaigFunction.d.ts +110 -4
- package/dist/SwaigFunction.js +151 -19
- package/dist/SwaigFunction.js.map +1 -1
- package/dist/SwmlBuilder.d.ts +68 -19
- package/dist/SwmlBuilder.js +74 -38
- package/dist/SwmlBuilder.js.map +1 -1
- package/dist/SwmlVerbMethods.generated.d.ts +41 -4
- package/dist/SwmlVerbMethods.generated.js +1 -1
- package/dist/TypeInference.d.ts +39 -3
- package/dist/TypeInference.js +256 -16
- package/dist/TypeInference.js.map +1 -1
- package/dist/WebService.d.ts +79 -11
- package/dist/WebService.js +320 -109
- package/dist/WebService.js.map +1 -1
- package/dist/WebhookMiddleware.d.ts +119 -0
- package/dist/WebhookMiddleware.js +271 -0
- package/dist/WebhookMiddleware.js.map +1 -0
- package/dist/WebhookValidator.d.ts +92 -0
- package/dist/WebhookValidator.js +409 -0
- package/dist/WebhookValidator.js.map +1 -0
- package/dist/agents/BedrockAgent.d.ts +162 -0
- package/dist/agents/BedrockAgent.js +323 -0
- package/dist/agents/BedrockAgent.js.map +1 -0
- package/dist/ai-chat/AIChatClient.d.ts +316 -0
- package/dist/ai-chat/AIChatClient.js +535 -0
- package/dist/ai-chat/AIChatClient.js.map +1 -0
- package/dist/ai-chat/ChatGateway.d.ts +408 -0
- package/dist/ai-chat/ChatGateway.js +747 -0
- package/dist/ai-chat/ChatGateway.js.map +1 -0
- package/dist/ai-chat/HandoffRouter.d.ts +282 -0
- package/dist/ai-chat/HandoffRouter.js +518 -0
- package/dist/ai-chat/HandoffRouter.js.map +1 -0
- package/dist/ai-chat/index.d.ts +12 -0
- package/dist/ai-chat/index.js +10 -0
- package/dist/ai-chat/index.js.map +1 -0
- package/dist/capabilities.d.ts +64 -0
- package/dist/capabilities.js +95 -0
- package/dist/capabilities.js.map +1 -0
- package/dist/cli/agent-loader.d.ts +32 -6
- package/dist/cli/agent-loader.js +95 -34
- package/dist/cli/agent-loader.js.map +1 -1
- package/dist/cli/datamap-exec.d.ts +175 -0
- package/dist/cli/datamap-exec.js +1284 -0
- package/dist/cli/datamap-exec.js.map +1 -0
- package/dist/cli/function-args.d.ts +40 -0
- package/dist/cli/function-args.js +146 -0
- package/dist/cli/function-args.js.map +1 -0
- package/dist/cli/simulation.d.ts +99 -0
- package/dist/cli/simulation.js +490 -0
- package/dist/cli/simulation.js.map +1 -0
- package/dist/cli/swaig-test.d.ts +12 -25
- package/dist/cli/swaig-test.js +935 -367
- package/dist/cli/swaig-test.js.map +1 -1
- package/dist/cli/tsdocs/agentsNote.d.ts +30 -0
- package/dist/cli/tsdocs/agentsNote.js +92 -0
- package/dist/cli/tsdocs/agentsNote.js.map +1 -0
- package/dist/cli/tsdocs/api-index.json +1 -0
- package/dist/cli/tsdocs/api.d.ts +91 -0
- package/dist/cli/tsdocs/api.js +372 -0
- package/dist/cli/tsdocs/api.js.map +1 -0
- package/dist/cli/tsdocs/apiTypes.d.ts +64 -0
- package/dist/cli/tsdocs/apiTypes.js +7 -0
- package/dist/cli/tsdocs/apiTypes.js.map +1 -0
- package/dist/cli/tsdocs/bin.d.ts +6 -0
- package/dist/cli/tsdocs/bin.js +23 -0
- package/dist/cli/tsdocs/bin.js.map +1 -0
- package/dist/cli/tsdocs/files.d.ts +108 -0
- package/dist/cli/tsdocs/files.js +287 -0
- package/dist/cli/tsdocs/files.js.map +1 -0
- package/dist/cli/tsdocs/index.d.ts +28 -0
- package/dist/cli/tsdocs/index.js +307 -0
- package/dist/cli/tsdocs/index.js.map +1 -0
- package/dist/cli/tsdocs/live.d.ts +41 -0
- package/dist/cli/tsdocs/live.js +202 -0
- package/dist/cli/tsdocs/live.js.map +1 -0
- package/dist/cli/tsdocs/locate.d.ts +51 -0
- package/dist/cli/tsdocs/locate.js +75 -0
- package/dist/cli/tsdocs/locate.js.map +1 -0
- package/dist/cli/tsdocs/render.d.ts +75 -0
- package/dist/cli/tsdocs/render.js +323 -0
- package/dist/cli/tsdocs/render.js.map +1 -0
- package/dist/cli/tsdocs/topics.d.ts +43 -0
- package/dist/cli/tsdocs/topics.js +916 -0
- package/dist/cli/tsdocs/topics.js.map +1 -0
- package/dist/index.d.ts +102 -39
- package/dist/index.js +107 -30
- package/dist/index.js.map +1 -1
- package/dist/livewire/index.d.ts +229 -155
- package/dist/livewire/index.js +298 -195
- package/dist/livewire/index.js.map +1 -1
- package/dist/llms.txt +65 -0
- package/dist/prefabs/CallSessionStore.d.ts +52 -0
- package/dist/prefabs/CallSessionStore.js +72 -0
- package/dist/prefabs/CallSessionStore.js.map +1 -0
- package/dist/prefabs/ConciergeAgent.d.ts +13 -1
- package/dist/prefabs/ConciergeAgent.js +36 -31
- package/dist/prefabs/ConciergeAgent.js.map +1 -1
- package/dist/prefabs/FAQBotAgent.d.ts +9 -1
- package/dist/prefabs/FAQBotAgent.js +113 -47
- package/dist/prefabs/FAQBotAgent.js.map +1 -1
- package/dist/prefabs/InfoGathererAgent.d.ts +22 -3
- package/dist/prefabs/InfoGathererAgent.js +67 -58
- package/dist/prefabs/InfoGathererAgent.js.map +1 -1
- package/dist/prefabs/ReceptionistAgent.d.ts +16 -5
- package/dist/prefabs/ReceptionistAgent.js +39 -31
- package/dist/prefabs/ReceptionistAgent.js.map +1 -1
- package/dist/prefabs/SurveyAgent.d.ts +22 -4
- package/dist/prefabs/SurveyAgent.js +84 -68
- package/dist/prefabs/SurveyAgent.js.map +1 -1
- package/dist/relay/Action.d.ts +44 -21
- package/dist/relay/Action.js +75 -17
- package/dist/relay/Action.js.map +1 -1
- package/dist/relay/Call.d.ts +312 -47
- package/dist/relay/Call.js +369 -18
- package/dist/relay/Call.js.map +1 -1
- package/dist/relay/Deferred.js.map +1 -1
- package/dist/relay/Message.d.ts +26 -5
- package/dist/relay/Message.js +36 -6
- package/dist/relay/Message.js.map +1 -1
- package/dist/relay/RelayClient.d.ts +79 -14
- package/dist/relay/RelayClient.js +268 -61
- package/dist/relay/RelayClient.js.map +1 -1
- package/dist/relay/RelayError.d.ts +14 -0
- package/dist/relay/RelayError.js +15 -0
- package/dist/relay/RelayError.js.map +1 -1
- package/dist/relay/RelayEvent.d.ts +96 -65
- package/dist/relay/RelayEvent.js +85 -56
- package/dist/relay/RelayEvent.js.map +1 -1
- package/dist/relay/closedSets.d.ts +111 -0
- package/dist/relay/closedSets.js +83 -0
- package/dist/relay/closedSets.js.map +1 -0
- package/dist/relay/index.d.ts +3 -1
- package/dist/relay/index.js +1 -0
- package/dist/relay/index.js.map +1 -1
- package/dist/relay/normalize.js.map +1 -1
- package/dist/relay/protocol.types.generated.d.ts +1195 -0
- package/dist/relay/protocol.types.generated.js +8 -0
- package/dist/relay/protocol.types.generated.js.map +1 -0
- package/dist/relay/types.d.ts +81 -39
- package/dist/rest/HttpClient.d.ts +36 -8
- package/dist/rest/HttpClient.js +233 -40
- package/dist/rest/HttpClient.js.map +1 -1
- package/dist/rest/RequestOptions.d.ts +110 -0
- package/dist/rest/RequestOptions.js +113 -0
- package/dist/rest/RequestOptions.js.map +1 -0
- package/dist/rest/RestError.d.ts +52 -6
- package/dist/rest/RestError.js +89 -5
- package/dist/rest/RestError.js.map +1 -1
- package/dist/rest/base/CrudResource.d.ts +16 -26
- package/dist/rest/base/CrudResource.js +19 -33
- package/dist/rest/base/CrudResource.js.map +1 -1
- package/dist/rest/base/CrudWithAddresses.d.ts +3 -2
- package/dist/rest/base/CrudWithAddresses.js +2 -2
- package/dist/rest/base/CrudWithAddresses.js.map +1 -1
- package/dist/rest/base/FabricResource.d.ts +15 -0
- package/dist/rest/base/FabricResource.js +16 -0
- package/dist/rest/base/FabricResource.js.map +1 -0
- package/dist/rest/base/ReadResource.d.ts +62 -0
- package/dist/rest/base/ReadResource.js +69 -0
- package/dist/rest/base/ReadResource.js.map +1 -0
- package/dist/rest/callHandler.d.ts +63 -0
- package/dist/rest/callHandler.js +62 -0
- package/dist/rest/callHandler.js.map +1 -0
- package/dist/rest/index.d.ts +19 -76
- package/dist/rest/index.js +49 -120
- package/dist/rest/index.js.map +1 -1
- package/dist/rest/namespaces/_client_tree_generated.d.ts +103 -0
- package/dist/rest/namespaces/_client_tree_generated.js +171 -0
- package/dist/rest/namespaces/_client_tree_generated.js.map +1 -0
- package/dist/rest/namespaces/addresses.d.ts +3 -42
- package/dist/rest/namespaces/addresses.js +3 -50
- package/dist/rest/namespaces/addresses.js.map +1 -1
- package/dist/rest/namespaces/calling.d.ts +6 -378
- package/dist/rest/namespaces/calling.js +6 -473
- package/dist/rest/namespaces/calling.js.map +1 -1
- package/dist/rest/namespaces/calling.resources.generated.d.ts +165 -0
- package/dist/rest/namespaces/calling.resources.generated.js +477 -0
- package/dist/rest/namespaces/calling.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/calling.types.generated.d.ts +2917 -0
- package/dist/rest/namespaces/calling.types.generated.js +8 -0
- package/dist/rest/namespaces/calling.types.generated.js.map +1 -0
- package/dist/rest/namespaces/chat.d.ts +4 -18
- package/dist/rest/namespaces/chat.js +4 -21
- package/dist/rest/namespaces/chat.js.map +1 -1
- package/dist/rest/namespaces/chat.resources.generated.d.ts +12 -0
- package/dist/rest/namespaces/chat.resources.generated.js +29 -0
- package/dist/rest/namespaces/chat.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/chat.types.generated.d.ts +56 -0
- package/dist/rest/namespaces/chat.types.generated.js +8 -0
- package/dist/rest/namespaces/chat.types.generated.js.map +1 -0
- package/dist/rest/namespaces/datasphere.d.ts +6 -62
- package/dist/rest/namespaces/datasphere.js +6 -72
- package/dist/rest/namespaces/datasphere.js.map +1 -1
- package/dist/rest/namespaces/datasphere.resources.generated.d.ts +25 -0
- package/dist/rest/namespaces/datasphere.resources.generated.js +50 -0
- package/dist/rest/namespaces/datasphere.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/datasphere.types.generated.d.ts +180 -0
- package/dist/rest/namespaces/datasphere.types.generated.js +8 -0
- package/dist/rest/namespaces/datasphere.types.generated.js.map +1 -0
- package/dist/rest/namespaces/fabric.d.ts +11 -273
- package/dist/rest/namespaces/fabric.js +14 -363
- package/dist/rest/namespaces/fabric.js.map +1 -1
- package/dist/rest/namespaces/fabric.resources.generated.d.ts +195 -0
- package/dist/rest/namespaces/fabric.resources.generated.js +399 -0
- package/dist/rest/namespaces/fabric.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/fabric.types.generated.d.ts +4557 -0
- package/dist/rest/namespaces/fabric.types.generated.js +8 -0
- package/dist/rest/namespaces/fabric.types.generated.js.map +1 -0
- package/dist/rest/namespaces/fax.resources.generated.d.ts +6 -0
- package/dist/rest/namespaces/fax.resources.generated.js +14 -0
- package/dist/rest/namespaces/fax.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/fax.types.generated.d.ts +131 -0
- package/dist/rest/namespaces/fax.types.generated.js +8 -0
- package/dist/rest/namespaces/fax.types.generated.js.map +1 -0
- package/dist/rest/namespaces/imported-numbers.d.ts +3 -18
- package/dist/rest/namespaces/imported-numbers.js +3 -21
- package/dist/rest/namespaces/imported-numbers.js.map +1 -1
- package/dist/rest/namespaces/logs.d.ts +10 -101
- package/dist/rest/namespaces/logs.js +10 -128
- package/dist/rest/namespaces/logs.js.map +1 -1
- package/dist/rest/namespaces/logs.resources.generated.d.ts +9 -0
- package/dist/rest/namespaces/logs.resources.generated.js +17 -0
- package/dist/rest/namespaces/logs.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/logs.types.generated.d.ts +131 -0
- package/dist/rest/namespaces/logs.types.generated.js +8 -0
- package/dist/rest/namespaces/logs.types.generated.js.map +1 -0
- package/dist/rest/namespaces/lookup.d.ts +3 -25
- package/dist/rest/namespaces/lookup.js +3 -27
- package/dist/rest/namespaces/lookup.js.map +1 -1
- package/dist/rest/namespaces/message.resources.generated.d.ts +6 -0
- package/dist/rest/namespaces/message.resources.generated.js +14 -0
- package/dist/rest/namespaces/message.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/message.types.generated.d.ts +122 -0
- package/dist/rest/namespaces/message.types.generated.js +8 -0
- package/dist/rest/namespaces/message.types.generated.js.map +1 -0
- package/dist/rest/namespaces/messages.resources.generated.d.ts +18 -0
- package/dist/rest/namespaces/messages.resources.generated.js +44 -0
- package/dist/rest/namespaces/messages.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/messages.types.generated.d.ts +99 -0
- package/dist/rest/namespaces/messages.types.generated.js +8 -0
- package/dist/rest/namespaces/messages.types.generated.js.map +1 -0
- package/dist/rest/namespaces/mfa.d.ts +3 -42
- package/dist/rest/namespaces/mfa.js +3 -49
- package/dist/rest/namespaces/mfa.js.map +1 -1
- package/dist/rest/namespaces/number-groups.d.ts +3 -45
- package/dist/rest/namespaces/number-groups.js +3 -53
- package/dist/rest/namespaces/number-groups.js.map +1 -1
- package/dist/rest/namespaces/phone-numbers.d.ts +9 -57
- package/dist/rest/namespaces/phone-numbers.js +9 -63
- package/dist/rest/namespaces/phone-numbers.js.map +1 -1
- package/dist/rest/namespaces/project.d.ts +6 -42
- package/dist/rest/namespaces/project.js +6 -51
- package/dist/rest/namespaces/project.js.map +1 -1
- package/dist/rest/namespaces/project.resources.generated.d.ts +17 -0
- package/dist/rest/namespaces/project.resources.generated.js +44 -0
- package/dist/rest/namespaces/project.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/project.types.generated.d.ts +58 -0
- package/dist/rest/namespaces/project.types.generated.js +8 -0
- package/dist/rest/namespaces/project.types.generated.js.map +1 -0
- package/dist/rest/namespaces/projects.resources.generated.d.ts +12 -0
- package/dist/rest/namespaces/projects.resources.generated.js +25 -0
- package/dist/rest/namespaces/projects.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/projects.types.generated.d.ts +77 -0
- package/dist/rest/namespaces/projects.types.generated.js +8 -0
- package/dist/rest/namespaces/projects.types.generated.js.map +1 -0
- package/dist/rest/namespaces/pubsub.d.ts +4 -18
- package/dist/rest/namespaces/pubsub.js +4 -21
- package/dist/rest/namespaces/pubsub.js.map +1 -1
- package/dist/rest/namespaces/pubsub.resources.generated.d.ts +12 -0
- package/dist/rest/namespaces/pubsub.resources.generated.js +29 -0
- package/dist/rest/namespaces/pubsub.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/pubsub.types.generated.d.ts +60 -0
- package/dist/rest/namespaces/pubsub.types.generated.js +8 -0
- package/dist/rest/namespaces/pubsub.types.generated.js.map +1 -0
- package/dist/rest/namespaces/queues.d.ts +3 -38
- package/dist/rest/namespaces/queues.js +3 -44
- package/dist/rest/namespaces/queues.js.map +1 -1
- package/dist/rest/namespaces/recordings.d.ts +3 -34
- package/dist/rest/namespaces/recordings.js +3 -40
- package/dist/rest/namespaces/recordings.js.map +1 -1
- package/dist/rest/namespaces/registry.d.ts +6 -137
- package/dist/rest/namespaces/registry.js +6 -173
- package/dist/rest/namespaces/registry.js.map +1 -1
- package/dist/rest/namespaces/relay-rest.resources.generated.d.ts +168 -0
- package/dist/rest/namespaces/relay-rest.resources.generated.js +430 -0
- package/dist/rest/namespaces/relay-rest.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/relay-rest.types.generated.d.ts +1906 -0
- package/dist/rest/namespaces/relay-rest.types.generated.js +8 -0
- package/dist/rest/namespaces/relay-rest.types.generated.js.map +1 -0
- package/dist/rest/namespaces/short-codes.d.ts +3 -35
- package/dist/rest/namespaces/short-codes.js +3 -41
- package/dist/rest/namespaces/short-codes.js.map +1 -1
- package/dist/rest/namespaces/sip-profile.d.ts +3 -24
- package/dist/rest/namespaces/sip-profile.js +3 -29
- package/dist/rest/namespaces/sip-profile.js.map +1 -1
- package/dist/rest/namespaces/verified-callers.d.ts +3 -29
- package/dist/rest/namespaces/verified-callers.js +3 -34
- package/dist/rest/namespaces/verified-callers.js.map +1 -1
- package/dist/rest/namespaces/video.d.ts +10 -227
- package/dist/rest/namespaces/video.js +10 -288
- package/dist/rest/namespaces/video.js.map +1 -1
- package/dist/rest/namespaces/video.resources.generated.d.ts +81 -0
- package/dist/rest/namespaces/video.resources.generated.js +171 -0
- package/dist/rest/namespaces/video.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/video.types.generated.d.ts +771 -0
- package/dist/rest/namespaces/video.types.generated.js +8 -0
- package/dist/rest/namespaces/video.types.generated.js.map +1 -0
- package/dist/rest/namespaces/voice.resources.generated.d.ts +9 -0
- package/dist/rest/namespaces/voice.resources.generated.js +17 -0
- package/dist/rest/namespaces/voice.resources.generated.js.map +1 -0
- package/dist/rest/namespaces/voice.types.generated.d.ts +214 -0
- package/dist/rest/namespaces/voice.types.generated.js +8 -0
- package/dist/rest/namespaces/voice.types.generated.js.map +1 -0
- package/dist/rest/pagination.d.ts +5 -1
- package/dist/rest/pagination.js +40 -20
- package/dist/rest/pagination.js.map +1 -1
- package/dist/rest/types.d.ts +21 -3
- package/dist/schema.json +299 -98
- package/dist/skills/SkillBase.d.ts +97 -20
- package/dist/skills/SkillBase.js +100 -17
- package/dist/skills/SkillBase.js.map +1 -1
- package/dist/skills/SkillManager.d.ts +66 -13
- package/dist/skills/SkillManager.js +101 -20
- package/dist/skills/SkillManager.js.map +1 -1
- package/dist/skills/SkillName.d.ts +44 -0
- package/dist/skills/SkillName.js +2 -0
- package/dist/skills/SkillName.js.map +1 -0
- package/dist/skills/SkillRegistry.d.ts +47 -7
- package/dist/skills/SkillRegistry.js +85 -13
- package/dist/skills/SkillRegistry.js.map +1 -1
- package/dist/skills/builtin/api_ninjas_trivia.d.ts +3 -1
- package/dist/skills/builtin/api_ninjas_trivia.js +24 -8
- package/dist/skills/builtin/api_ninjas_trivia.js.map +1 -1
- package/dist/skills/builtin/ask_claude.d.ts +14 -4
- package/dist/skills/builtin/ask_claude.js +36 -14
- package/dist/skills/builtin/ask_claude.js.map +1 -1
- package/dist/skills/builtin/claude_skills.d.ts +3 -1
- package/dist/skills/builtin/claude_skills.js +19 -15
- package/dist/skills/builtin/claude_skills.js.map +1 -1
- package/dist/skills/builtin/custom_skills.d.ts +10 -3
- package/dist/skills/builtin/custom_skills.js +26 -9
- package/dist/skills/builtin/custom_skills.js.map +1 -1
- package/dist/skills/builtin/datasphere.d.ts +5 -3
- package/dist/skills/builtin/datasphere.js +34 -25
- package/dist/skills/builtin/datasphere.js.map +1 -1
- package/dist/skills/builtin/datasphere_serverless.d.ts +12 -1
- package/dist/skills/builtin/datasphere_serverless.js +17 -6
- package/dist/skills/builtin/datasphere_serverless.js.map +1 -1
- package/dist/skills/builtin/datetime.d.ts +3 -1
- package/dist/skills/builtin/datetime.js +11 -7
- package/dist/skills/builtin/datetime.js.map +1 -1
- package/dist/skills/builtin/google_maps.d.ts +29 -14
- package/dist/skills/builtin/google_maps.js +89 -269
- package/dist/skills/builtin/google_maps.js.map +1 -1
- package/dist/skills/builtin/index.d.ts +8 -6
- package/dist/skills/builtin/index.js +8 -6
- package/dist/skills/builtin/index.js.map +1 -1
- package/dist/skills/builtin/info_gatherer.d.ts +4 -2
- package/dist/skills/builtin/info_gatherer.js +20 -12
- package/dist/skills/builtin/info_gatherer.js.map +1 -1
- package/dist/skills/builtin/joke.d.ts +16 -6
- package/dist/skills/builtin/joke.js +54 -37
- package/dist/skills/builtin/joke.js.map +1 -1
- package/dist/skills/builtin/math.d.ts +3 -1
- package/dist/skills/builtin/math.js +14 -9
- package/dist/skills/builtin/math.js.map +1 -1
- package/dist/skills/builtin/mcp_gateway.d.ts +34 -5
- package/dist/skills/builtin/mcp_gateway.js +77 -46
- package/dist/skills/builtin/mcp_gateway.js.map +1 -1
- package/dist/skills/builtin/native_vector_search.d.ts +19 -5
- package/dist/skills/builtin/native_vector_search.js +179 -61
- package/dist/skills/builtin/native_vector_search.js.map +1 -1
- package/dist/skills/builtin/play_background_file.d.ts +21 -15
- package/dist/skills/builtin/play_background_file.js +43 -122
- package/dist/skills/builtin/play_background_file.js.map +1 -1
- package/dist/skills/builtin/spider.d.ts +46 -1
- package/dist/skills/builtin/spider.js +317 -101
- package/dist/skills/builtin/spider.js.map +1 -1
- package/dist/skills/builtin/swml_transfer.d.ts +14 -3
- package/dist/skills/builtin/swml_transfer.js +45 -24
- package/dist/skills/builtin/swml_transfer.js.map +1 -1
- package/dist/skills/builtin/weather_api.d.ts +24 -13
- package/dist/skills/builtin/weather_api.js +57 -31
- package/dist/skills/builtin/weather_api.js.map +1 -1
- package/dist/skills/builtin/web_search.d.ts +48 -10
- package/dist/skills/builtin/web_search.js +301 -84
- package/dist/skills/builtin/web_search.js.map +1 -1
- package/dist/skills/builtin/wikipedia_search.d.ts +4 -2
- package/dist/skills/builtin/wikipedia_search.js +32 -14
- package/dist/skills/builtin/wikipedia_search.js.map +1 -1
- package/dist/skills/index.d.ts +3 -2
- package/dist/skills/index.js +1 -1
- package/dist/skills/index.js.map +1 -1
- package/dist/swml_verbs_generated.d.ts +2824 -0
- package/dist/swml_verbs_generated.js +12 -0
- package/dist/swml_verbs_generated.js.map +1 -0
- package/dist/types.d.ts +65 -6
- package/dist/web.d.ts +27 -0
- package/dist/web.js +20 -0
- package/dist/web.js.map +1 -0
- package/docs/agent-guide.md +1034 -0
- package/docs/ai_chat.md +560 -0
- package/docs/api-reference.md +3534 -0
- package/docs/architecture.md +846 -0
- package/docs/bedrock_agent.md +411 -0
- package/docs/cli-guide.md +403 -0
- package/docs/cloud_functions_guide.md +399 -0
- package/docs/configuration.md +548 -0
- package/docs/contexts-guide.md +884 -0
- package/docs/datamap-guide.md +1160 -0
- package/docs/developer_pain_points.md +548 -0
- package/docs/llm_parameters.md +298 -0
- package/docs/mcp_gateway_reference.md +354 -0
- package/docs/mcp_integration.md +162 -0
- package/docs/pgi_agent_guide.md +1820 -0
- package/docs/prefabs-guide.md +815 -0
- package/docs/programmatically_governed_inference.md +301 -0
- package/docs/sdk_features.md +501 -0
- package/docs/security.md +559 -0
- package/docs/serverless-guide.md +520 -0
- package/docs/skills-guide.md +1223 -0
- package/docs/skills-system.md +405 -0
- package/docs/skills_parameter_schema.md +417 -0
- package/docs/swaig-reference.md +1938 -0
- package/docs/swml_service_guide.md +573 -0
- package/docs/third_party_skills.md +410 -0
- package/docs/web_service.md +666 -0
- package/examples/README.md +202 -0
- package/examples/advanced-datamap.ts +108 -0
- package/examples/advanced-dynamic-config.ts +70 -0
- package/examples/auto-vivified.ts +95 -0
- package/examples/call-flow.ts +65 -0
- package/examples/comprehensive-dynamic.ts +125 -0
- package/examples/contexts-steps.ts +65 -0
- package/examples/custom-path.ts +45 -0
- package/examples/datamap-tools.ts +51 -0
- package/examples/datasphere-multi-instance.ts +66 -0
- package/examples/datasphere-serverless-env.ts +87 -0
- package/examples/datasphere-webhook-env.ts +65 -0
- package/examples/datasphere.ts +56 -0
- package/examples/declarative.ts +100 -0
- package/examples/dynamic-config.ts +54 -0
- package/examples/dynamic-info-gatherer.ts +69 -0
- package/examples/dynamic-swml-service.ts +67 -0
- package/examples/gather-info.ts +95 -0
- package/examples/gather-per-question-functions-demo.ts +139 -0
- package/examples/joke-agent.ts +34 -0
- package/examples/kubernetes-agent.ts +45 -0
- package/examples/llm-params.ts +75 -0
- package/examples/mcp-agent.ts +89 -0
- package/examples/mcp-gateway.ts +81 -0
- package/examples/multi-agent.ts +80 -0
- package/examples/multi-endpoint.ts +79 -0
- package/examples/pgi/case-domain.ts +114 -0
- package/examples/pgi/serve.ts +16 -0
- package/examples/pgi/support-agent.ts +265 -0
- package/examples/pom-prompt.ts +70 -0
- package/examples/prefab-concierge.ts +42 -0
- package/examples/prefab-faq.ts +58 -0
- package/examples/prefab-info-gatherer.ts +33 -0
- package/examples/prefab-receptionist.ts +36 -0
- package/examples/prefab-survey.ts +69 -0
- package/examples/quickstart-agent.ts +29 -0
- package/examples/quickstart-relay.ts +25 -0
- package/examples/quickstart-rest.ts +25 -0
- package/examples/record-call.ts +65 -0
- package/examples/relay-demo.ts +33 -0
- package/examples/room-and-sip.ts +93 -0
- package/examples/serverless-lambda.ts +43 -0
- package/examples/session-state.ts +83 -0
- package/examples/simple-agent.ts +36 -0
- package/examples/simple-static.ts +29 -0
- package/examples/skills-demo.ts +41 -0
- package/examples/step-function-inheritance-demo.ts +124 -0
- package/examples/swaig-features.ts +193 -0
- package/examples/swml-service-routing.ts +66 -0
- package/examples/swml-service.ts +38 -0
- package/examples/swmlservice_ai_sidecar.ts +104 -0
- package/examples/swmlservice_swaig_standalone.ts +79 -0
- package/examples/tap.ts +67 -0
- package/examples/typed-tools.ts +68 -0
- package/examples/verb-methods.ts +45 -0
- package/examples/web-search-multi-instance.ts +75 -0
- package/examples/web-search.ts +38 -0
- package/examples/wikipedia.ts +36 -0
- package/livewire/README.md +164 -0
- package/livewire/docs/migration-guide.md +251 -0
- package/livewire/examples/livewire-basic-agent.ts +74 -0
- package/livewire/examples/livewire-handoff.ts +196 -0
- package/livewire/examples/livewire-multi-tool.ts +121 -0
- package/package.json +88 -13
- package/relay/README.md +101 -0
- package/relay/docs/call-methods.md +622 -0
- package/relay/docs/client-reference.md +281 -0
- package/relay/docs/events.md +193 -0
- package/relay/docs/getting-started.md +221 -0
- package/relay/docs/guide.md +386 -0
- package/relay/docs/messaging.md +246 -0
- package/relay/examples/relay-inbound.ts +66 -0
- package/relay/examples/relay-messaging.ts +74 -0
- package/relay/examples/relay-outbound.ts +76 -0
- package/rest/README.md +141 -0
- package/rest/docs/calling.md +313 -0
- package/rest/docs/client-reference.md +149 -0
- package/rest/docs/fabric.md +192 -0
- package/rest/docs/getting-started.md +138 -0
- package/rest/docs/guide.md +421 -0
- package/rest/docs/namespaces.md +367 -0
- package/rest/docs/phone-binding.md +116 -0
- package/rest/examples/rest-10dlc-registration.ts +177 -0
- package/rest/examples/rest-bind-phone-to-swml-webhook.ts +66 -0
- package/rest/examples/rest-calling-ivr-and-ai.ts +136 -0
- package/rest/examples/rest-calling-play-and-record.ts +98 -0
- package/rest/examples/rest-client.ts +59 -0
- package/rest/examples/rest-datasphere-search.ts +81 -0
- package/rest/examples/rest-fabric-conferences-and-routing.ts +147 -0
- package/rest/examples/rest-fabric-subscribers-and-sip.ts +106 -0
- package/rest/examples/rest-fabric-swml-and-callflows.ts +101 -0
- package/rest/examples/rest-manage-resources.ts +64 -0
- package/rest/examples/rest-phone-number-management.ts +200 -0
- package/rest/examples/rest-queues-mfa-and-recordings.ts +143 -0
- package/rest/examples/rest-video-rooms.ts +186 -0
- package/tutorial/fred/fred.ts +218 -0
- package/tutorial/fred/tutorial/01-introduction.md +141 -0
- package/tutorial/fred/tutorial/02-setup.md +304 -0
- package/tutorial/fred/tutorial/03-basic-agent.md +284 -0
- package/tutorial/fred/tutorial/04-wikipedia-skill.md +298 -0
- package/tutorial/fred/tutorial/05-custom-functions.md +347 -0
- package/tutorial/fred/tutorial/06-running-testing.md +642 -0
- package/tutorial/fred/tutorial/README.md +56 -0
- package/tutorial/fred/tutorial/appendix-complete-code.md +632 -0
- package/tutorial/fred/tutorial/appendix-docker-deployment.md +504 -0
- package/tutorial/full-guardrails-agent/handlers.ts +436 -0
- package/tutorial/full-guardrails-agent/penny.ts +305 -0
- package/tutorial/full-guardrails-agent/reservations.ts +1109 -0
- package/tutorial/full-guardrails-agent/tutorial/01-why-guardrails.md +149 -0
- package/tutorial/full-guardrails-agent/tutorial/02-design-first.md +139 -0
- package/tutorial/full-guardrails-agent/tutorial/03-rules-first.md +422 -0
- package/tutorial/full-guardrails-agent/tutorial/04-the-shell.md +238 -0
- package/tutorial/full-guardrails-agent/tutorial/05-steps-and-scoping.md +247 -0
- package/tutorial/full-guardrails-agent/tutorial/06-tools-that-decide.md +467 -0
- package/tutorial/full-guardrails-agent/tutorial/07-gather-and-projection.md +160 -0
- package/tutorial/full-guardrails-agent/tutorial/08-the-verification-gate.md +333 -0
- package/tutorial/full-guardrails-agent/tutorial/09-people-and-endings.md +313 -0
- package/tutorial/full-guardrails-agent/tutorial/10-testing-and-running.md +387 -0
- package/tutorial/full-guardrails-agent/tutorial/README.md +106 -0
- package/tutorial/full-guardrails-agent/tutorial/appendix-complete-code.md +2207 -0
- package/tutorial/full-guardrails-agent/tutorial/appendix-deployment.md +191 -0
- package/tutorial/full-guardrails-agent/tutorial/appendix-technique-map.md +117 -0
- package/tutorial/full-guardrails-agent/workflow.ts +219 -0
- package/tutorial/multi_agents/README.md +175 -0
- package/tutorial/multi_agents/advanced_agent.ts +395 -0
- package/tutorial/multi_agents/extending_agents.ts +641 -0
- package/tutorial/multi_agents/knowledge.ts +59 -0
- package/tutorial/multi_agents/lesson1_first_agent.md +368 -0
- package/tutorial/multi_agents/lesson2_knowledge_bases.md +404 -0
- package/tutorial/multi_agents/lesson3_multi_agent_systems.md +621 -0
- package/tutorial/multi_agents/lesson4_advanced_features.md +805 -0
- package/tutorial/multi_agents/lesson5_extending_agents.md +856 -0
- package/tutorial/multi_agents/pc_builder.ts +529 -0
- package/tutorial/multi_agents/sales_agent.ts +88 -0
- package/tutorial/multi_agents/sales_agent_with_search.ts +116 -0
- package/tutorial/multi_agents/sales_knowledge.md +139 -0
- package/tutorial/multi_agents/support_knowledge.md +256 -0
- package/dist/cli/mock-data.d.ts +0 -37
- package/dist/cli/mock-data.js +0 -80
- package/dist/cli/mock-data.js.map +0 -1
- package/dist/generateVerbTypes.d.ts +0 -11
- package/dist/generateVerbTypes.js +0 -265
- package/dist/generateVerbTypes.js.map +0 -1
- package/dist/rest/namespaces/compat.d.ts +0 -585
- package/dist/rest/namespaces/compat.js +0 -735
- package/dist/rest/namespaces/compat.js.map +0 -1
|
@@ -11,23 +11,16 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Multiple instances are supported; use `tool_name` in config to differentiate.
|
|
13
13
|
*/
|
|
14
|
-
import { SkillBase } from '../SkillBase.js';
|
|
14
|
+
import { SkillBase, defineSkillTool } from '../SkillBase.js';
|
|
15
15
|
import { FunctionResult } from '../../FunctionResult.js';
|
|
16
|
+
import { _publicFetch, _RedirectRefused } from '../../PublicFetch.js';
|
|
17
|
+
import { _RobotsRules } from '../../RobotsTxt.js';
|
|
16
18
|
import { resolveAndValidateUrl, validateUrl, MAX_SKILL_INPUT_LENGTH } from '../../SecurityUtils.js';
|
|
17
19
|
import { getLogger } from '../../Logger.js';
|
|
18
|
-
import * as cheerio from 'cheerio';
|
|
19
20
|
const log = getLogger('SpiderSkill');
|
|
20
21
|
const WHITESPACE_REGEX = /\s+/g;
|
|
21
|
-
/**
|
|
22
|
-
const
|
|
23
|
-
/<script\b[^>]*>[\s\S]*?<\/script>/gi,
|
|
24
|
-
/<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
|
25
|
-
/<nav\b[^>]*>[\s\S]*?<\/nav>/gi,
|
|
26
|
-
/<header\b[^>]*>[\s\S]*?<\/header>/gi,
|
|
27
|
-
/<footer\b[^>]*>[\s\S]*?<\/footer>/gi,
|
|
28
|
-
/<aside\b[^>]*>[\s\S]*?<\/aside>/gi,
|
|
29
|
-
/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi,
|
|
30
|
-
];
|
|
22
|
+
/** Headers that carry credentials for one origin (as _publicFetch treats them). */
|
|
23
|
+
const ORIGIN_BOUND_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']);
|
|
31
24
|
/**
|
|
32
25
|
* Fast web scraping skill optimized for speed and token efficiency.
|
|
33
26
|
*
|
|
@@ -39,7 +32,9 @@ const STRIP_TAG_REGEXES = [
|
|
|
39
32
|
*
|
|
40
33
|
* @example
|
|
41
34
|
* ```ts
|
|
42
|
-
*
|
|
35
|
+
* import { AgentBase } from '@signalwire/sdk';
|
|
36
|
+
* const agent = new AgentBase({ name: 'demo', route: '/' });
|
|
37
|
+
* await agent.addSkillByName('spider', { max_pages: 5, max_depth: 2 });
|
|
43
38
|
* ```
|
|
44
39
|
*/
|
|
45
40
|
export class SpiderSkill extends SkillBase {
|
|
@@ -51,20 +46,64 @@ export class SpiderSkill extends SkillBase {
|
|
|
51
46
|
static REQUIRED_PACKAGES = ['cheerio'];
|
|
52
47
|
static REQUIRED_ENV_VARS = [];
|
|
53
48
|
static SUPPORTS_MULTIPLE_INSTANCES = true;
|
|
49
|
+
/**
|
|
50
|
+
* Each setting's default. The parameter schema and setup() both read this,
|
|
51
|
+
* so the schema can't advertise a default the skill doesn't use.
|
|
52
|
+
*/
|
|
53
|
+
static DEFAULTS = {
|
|
54
|
+
delay: 0.1,
|
|
55
|
+
concurrent_requests: 5,
|
|
56
|
+
timeout: 5,
|
|
57
|
+
max_pages: 1,
|
|
58
|
+
max_depth: 0,
|
|
59
|
+
extract_type: 'fast_text',
|
|
60
|
+
max_text_length: 3000,
|
|
61
|
+
clean_text: true,
|
|
62
|
+
cache_enabled: true,
|
|
63
|
+
follow_robots_txt: false,
|
|
64
|
+
user_agent: 'Spider/1.0 (SignalWire AI Agent)',
|
|
65
|
+
};
|
|
66
|
+
/** The extraction methods scrape_url implements. */
|
|
67
|
+
static EXTRACT_TYPES = [
|
|
68
|
+
'fast_text',
|
|
69
|
+
'markdown',
|
|
70
|
+
'structured',
|
|
71
|
+
];
|
|
72
|
+
/**
|
|
73
|
+
* Values the schema once listed that were never implemented. They have
|
|
74
|
+
* always worked as fast_text, so they still do, with a warning.
|
|
75
|
+
*/
|
|
76
|
+
static LEGACY_EXTRACT_TYPES = new Set([
|
|
77
|
+
'clean_text',
|
|
78
|
+
'full_text',
|
|
79
|
+
'html',
|
|
80
|
+
'custom',
|
|
81
|
+
]);
|
|
82
|
+
/** Seconds to keep a site's robots.txt rules; RFC 9309 allows up to 24 hours. */
|
|
83
|
+
static ROBOTS_TTL = 24 * 60 * 60;
|
|
54
84
|
static getParameterSchema() {
|
|
85
|
+
const d = SpiderSkill.DEFAULTS;
|
|
55
86
|
return {
|
|
56
87
|
...super.getParameterSchema(),
|
|
88
|
+
// The base schema's default is the skill name, but tool_name is a
|
|
89
|
+
// prefix that's empty when unset (Python skill.py:306 too).
|
|
90
|
+
tool_name: {
|
|
91
|
+
type: 'string',
|
|
92
|
+
description: 'Prefix for the tool names, as in <tool_name>_scrape_url, and for the instance key. ' +
|
|
93
|
+
'Unset, the tools keep their plain names.',
|
|
94
|
+
required: false,
|
|
95
|
+
},
|
|
57
96
|
delay: {
|
|
58
97
|
type: 'number',
|
|
59
98
|
description: 'Delay between requests in seconds',
|
|
60
|
-
default:
|
|
99
|
+
default: d.delay,
|
|
61
100
|
required: false,
|
|
62
101
|
min: 0,
|
|
63
102
|
},
|
|
64
103
|
concurrent_requests: {
|
|
65
104
|
type: 'integer',
|
|
66
|
-
description: '
|
|
67
|
-
default:
|
|
105
|
+
description: 'Deprecated, and has no effect: the spider fetches one page at a time',
|
|
106
|
+
default: d.concurrent_requests,
|
|
68
107
|
required: false,
|
|
69
108
|
min: 1,
|
|
70
109
|
max: 20,
|
|
@@ -72,7 +111,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
72
111
|
timeout: {
|
|
73
112
|
type: 'integer',
|
|
74
113
|
description: 'Request timeout in seconds',
|
|
75
|
-
default:
|
|
114
|
+
default: d.timeout,
|
|
76
115
|
required: false,
|
|
77
116
|
min: 1,
|
|
78
117
|
max: 60,
|
|
@@ -80,7 +119,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
80
119
|
max_pages: {
|
|
81
120
|
type: 'integer',
|
|
82
121
|
description: 'Maximum number of pages to scrape',
|
|
83
|
-
default:
|
|
122
|
+
default: d.max_pages,
|
|
84
123
|
required: false,
|
|
85
124
|
min: 1,
|
|
86
125
|
max: 100,
|
|
@@ -88,7 +127,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
88
127
|
max_depth: {
|
|
89
128
|
type: 'integer',
|
|
90
129
|
description: 'Maximum crawl depth (0 = single page only)',
|
|
91
|
-
default:
|
|
130
|
+
default: d.max_depth,
|
|
92
131
|
required: false,
|
|
93
132
|
min: 0,
|
|
94
133
|
max: 5,
|
|
@@ -96,14 +135,14 @@ export class SpiderSkill extends SkillBase {
|
|
|
96
135
|
extract_type: {
|
|
97
136
|
type: 'string',
|
|
98
137
|
description: 'Content extraction method',
|
|
99
|
-
default:
|
|
138
|
+
default: d.extract_type,
|
|
100
139
|
required: false,
|
|
101
|
-
enum: [
|
|
140
|
+
enum: [...SpiderSkill.EXTRACT_TYPES],
|
|
102
141
|
},
|
|
103
142
|
max_text_length: {
|
|
104
143
|
type: 'integer',
|
|
105
144
|
description: 'Maximum text length to return',
|
|
106
|
-
default:
|
|
145
|
+
default: d.max_text_length,
|
|
107
146
|
required: false,
|
|
108
147
|
min: 100,
|
|
109
148
|
max: 100000,
|
|
@@ -111,7 +150,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
111
150
|
clean_text: {
|
|
112
151
|
type: 'boolean',
|
|
113
152
|
description: 'Whether to clean extracted text',
|
|
114
|
-
default:
|
|
153
|
+
default: d.clean_text,
|
|
115
154
|
required: false,
|
|
116
155
|
},
|
|
117
156
|
selectors: {
|
|
@@ -131,7 +170,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
131
170
|
user_agent: {
|
|
132
171
|
type: 'string',
|
|
133
172
|
description: 'User agent string for requests',
|
|
134
|
-
default:
|
|
173
|
+
default: d.user_agent,
|
|
135
174
|
required: false,
|
|
136
175
|
},
|
|
137
176
|
headers: {
|
|
@@ -143,63 +182,72 @@ export class SpiderSkill extends SkillBase {
|
|
|
143
182
|
},
|
|
144
183
|
follow_robots_txt: {
|
|
145
184
|
type: 'boolean',
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
// True but the runtime and TS both use False — aligning the schema
|
|
149
|
-
// eliminates the cross-SDK contract mismatch.
|
|
150
|
-
description: 'Whether to respect robots.txt (default: false to match Python runtime)',
|
|
151
|
-
default: false,
|
|
185
|
+
description: "Skip pages that the site's robots.txt disallows for user_agent",
|
|
186
|
+
default: d.follow_robots_txt,
|
|
152
187
|
required: false,
|
|
153
188
|
},
|
|
154
189
|
cache_enabled: {
|
|
155
190
|
type: 'boolean',
|
|
156
191
|
description: 'Whether to cache scraped pages',
|
|
157
|
-
default:
|
|
192
|
+
default: d.cache_enabled,
|
|
158
193
|
required: false,
|
|
159
194
|
},
|
|
160
195
|
};
|
|
161
196
|
}
|
|
162
197
|
// Runtime state, populated in setup()
|
|
163
|
-
delay =
|
|
164
|
-
concurrentRequests =
|
|
165
|
-
timeout =
|
|
166
|
-
maxPages =
|
|
167
|
-
maxDepth =
|
|
168
|
-
extractType =
|
|
169
|
-
maxTextLength =
|
|
170
|
-
cleanText =
|
|
171
|
-
cacheEnabled =
|
|
172
|
-
followRobotsTxt =
|
|
173
|
-
userAgent =
|
|
198
|
+
delay = SpiderSkill.DEFAULTS.delay;
|
|
199
|
+
concurrentRequests = SpiderSkill.DEFAULTS.concurrent_requests;
|
|
200
|
+
timeout = SpiderSkill.DEFAULTS.timeout;
|
|
201
|
+
maxPages = SpiderSkill.DEFAULTS.max_pages;
|
|
202
|
+
maxDepth = SpiderSkill.DEFAULTS.max_depth;
|
|
203
|
+
extractType = SpiderSkill.DEFAULTS.extract_type;
|
|
204
|
+
maxTextLength = SpiderSkill.DEFAULTS.max_text_length;
|
|
205
|
+
cleanText = SpiderSkill.DEFAULTS.clean_text;
|
|
206
|
+
cacheEnabled = SpiderSkill.DEFAULTS.cache_enabled;
|
|
207
|
+
followRobotsTxt = SpiderSkill.DEFAULTS.follow_robots_txt;
|
|
208
|
+
userAgent = SpiderSkill.DEFAULTS.user_agent;
|
|
174
209
|
headers = {};
|
|
175
210
|
selectors = {};
|
|
176
211
|
compiledFollowPatterns = [];
|
|
177
212
|
cache = null;
|
|
178
213
|
cacheMaxSize = 100;
|
|
214
|
+
/** robots.txt rules per origin, with when they expire, when follow_robots_txt is on. */
|
|
215
|
+
robots = new Map();
|
|
216
|
+
// Lazily-loaded optional `cheerio` module, populated in setup(). Non-null
|
|
217
|
+
// for the lifetime of an initialized skill (setup() returns false if absent).
|
|
218
|
+
_cheerio;
|
|
179
219
|
getInstanceKey() {
|
|
180
220
|
const toolName = this.getConfig('tool_name', this.skillName);
|
|
181
221
|
return `${this.skillName}_${toolName}`;
|
|
182
222
|
}
|
|
183
223
|
async setup() {
|
|
224
|
+
// Load the optional cheerio dependency once. If it isn't installed, the
|
|
225
|
+
// skill cannot scrape — fail setup cleanly (validatePackages() also flags
|
|
226
|
+
// it). Cached on the instance so the sync extract helpers can use it.
|
|
227
|
+
try {
|
|
228
|
+
this._cheerio = await import('cheerio');
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
log.error('spider: the optional "cheerio" package is required for this skill but is not installed');
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
const d = SpiderSkill.DEFAULTS;
|
|
184
235
|
// Performance
|
|
185
|
-
this.delay = this.getConfig('delay',
|
|
186
|
-
this.concurrentRequests = this.getConfig('concurrent_requests',
|
|
187
|
-
this.timeout = this.getConfig('timeout',
|
|
236
|
+
this.delay = this.getConfig('delay', d.delay);
|
|
237
|
+
this.concurrentRequests = this.getConfig('concurrent_requests', d.concurrent_requests);
|
|
238
|
+
this.timeout = this.getConfig('timeout', d.timeout);
|
|
188
239
|
// Crawl limits
|
|
189
|
-
this.maxPages = this.getConfig('max_pages',
|
|
190
|
-
this.maxDepth = this.getConfig('max_depth',
|
|
240
|
+
this.maxPages = this.getConfig('max_pages', d.max_pages);
|
|
241
|
+
this.maxDepth = this.getConfig('max_depth', d.max_depth);
|
|
191
242
|
// Content processing
|
|
192
|
-
this.extractType = this.getConfig('extract_type',
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
// `self.params.get('max_text_length', 3000)` and schema defaults are not
|
|
196
|
-
// applied. Use 3000 for parity. Follow-up: reconcile Python schema vs init.
|
|
197
|
-
this.maxTextLength = this.getConfig('max_text_length', 3000);
|
|
198
|
-
this.cleanText = this.getConfig('clean_text', true);
|
|
243
|
+
this.extractType = this.getConfig('extract_type', d.extract_type);
|
|
244
|
+
this.maxTextLength = this.getConfig('max_text_length', d.max_text_length);
|
|
245
|
+
this.cleanText = this.getConfig('clean_text', d.clean_text);
|
|
199
246
|
// Features
|
|
200
|
-
this.cacheEnabled = this.getConfig('cache_enabled',
|
|
201
|
-
this.followRobotsTxt = this.getConfig('follow_robots_txt',
|
|
202
|
-
this.userAgent = this.getConfig('user_agent',
|
|
247
|
+
this.cacheEnabled = this.getConfig('cache_enabled', d.cache_enabled);
|
|
248
|
+
this.followRobotsTxt = this.getConfig('follow_robots_txt', d.follow_robots_txt);
|
|
249
|
+
this.userAgent = this.getConfig('user_agent', d.user_agent);
|
|
250
|
+
this.robots = new Map();
|
|
203
251
|
// Optional headers + user-agent merge
|
|
204
252
|
const headers = this.getConfig('headers', {}) ?? {};
|
|
205
253
|
this.headers = { ...headers, 'User-Agent': this.userAgent };
|
|
@@ -213,7 +261,22 @@ export class SpiderSkill extends SkillBase {
|
|
|
213
261
|
return false;
|
|
214
262
|
}
|
|
215
263
|
if (this.concurrentRequests < 1 || this.concurrentRequests > 20) {
|
|
216
|
-
log.error('spider: concurrent_requests must be between 1 and 20', {
|
|
264
|
+
log.error('spider: concurrent_requests must be between 1 and 20', {
|
|
265
|
+
concurrent_requests: this.concurrentRequests,
|
|
266
|
+
});
|
|
267
|
+
return false;
|
|
268
|
+
}
|
|
269
|
+
if ('concurrent_requests' in this.config) {
|
|
270
|
+
log.warn('spider: concurrent_requests is deprecated and has no effect: the spider fetches one page at a time');
|
|
271
|
+
}
|
|
272
|
+
// Validate the extraction method
|
|
273
|
+
const types = SpiderSkill.EXTRACT_TYPES.join(', ');
|
|
274
|
+
if (SpiderSkill.LEGACY_EXTRACT_TYPES.has(this.extractType)) {
|
|
275
|
+
log.warn(`spider: extract_type '${this.extractType}' was never implemented and works as fast_text; use one of ${types}`);
|
|
276
|
+
this.extractType = 'fast_text';
|
|
277
|
+
}
|
|
278
|
+
else if (!SpiderSkill.EXTRACT_TYPES.includes(this.extractType)) {
|
|
279
|
+
log.error(`spider: unknown extract_type '${this.extractType}'; use one of ${types}`);
|
|
217
280
|
return false;
|
|
218
281
|
}
|
|
219
282
|
if (this.maxPages < 1) {
|
|
@@ -267,8 +330,11 @@ export class SpiderSkill extends SkillBase {
|
|
|
267
330
|
getTools() {
|
|
268
331
|
const toolPrefix = this.getConfig('tool_name', '');
|
|
269
332
|
const prefix = toolPrefix ? `${toolPrefix}_` : '';
|
|
333
|
+
// The three handlers keep their defensive typeof/.trim()/URL-validation
|
|
334
|
+
// narrowing in the private methods: `url`/`start_url` are required, but the
|
|
335
|
+
// model can still emit an empty or wrong-typed value.
|
|
270
336
|
return [
|
|
271
|
-
{
|
|
337
|
+
defineSkillTool({
|
|
272
338
|
name: `${prefix}scrape_url`,
|
|
273
339
|
description: 'Extract text content from a single web page',
|
|
274
340
|
parameters: {
|
|
@@ -279,8 +345,8 @@ export class SpiderSkill extends SkillBase {
|
|
|
279
345
|
},
|
|
280
346
|
required: ['url'],
|
|
281
347
|
handler: async (args, rawData) => this._scrapeUrlHandler(args, rawData),
|
|
282
|
-
},
|
|
283
|
-
{
|
|
348
|
+
}),
|
|
349
|
+
defineSkillTool({
|
|
284
350
|
name: `${prefix}crawl_site`,
|
|
285
351
|
description: 'Crawl multiple pages starting from a URL',
|
|
286
352
|
parameters: {
|
|
@@ -291,8 +357,8 @@ export class SpiderSkill extends SkillBase {
|
|
|
291
357
|
},
|
|
292
358
|
required: ['start_url'],
|
|
293
359
|
handler: async (args, rawData) => this._crawlSiteHandler(args, rawData),
|
|
294
|
-
},
|
|
295
|
-
{
|
|
360
|
+
}),
|
|
361
|
+
defineSkillTool({
|
|
296
362
|
name: `${prefix}extract_structured_data`,
|
|
297
363
|
description: 'Extract specific data from a web page using selectors',
|
|
298
364
|
parameters: {
|
|
@@ -303,25 +369,168 @@ export class SpiderSkill extends SkillBase {
|
|
|
303
369
|
},
|
|
304
370
|
required: ['url'],
|
|
305
371
|
handler: async (args, rawData) => this._extractStructuredHandler(args, rawData),
|
|
306
|
-
},
|
|
372
|
+
}),
|
|
307
373
|
];
|
|
308
374
|
}
|
|
309
375
|
// ---------------------------------------------------------------------------
|
|
310
376
|
// Internal helpers
|
|
311
377
|
// ---------------------------------------------------------------------------
|
|
378
|
+
/**
|
|
379
|
+
* Redirect a target URL through `SPIDER_BASE_URL` when set. Used by tests
|
|
380
|
+
* to point every outbound fetch at a loopback fixture without requiring
|
|
381
|
+
* the test URL to resolve.
|
|
382
|
+
* Preserves the path (and any query/fragment) from the original target.
|
|
383
|
+
*/
|
|
384
|
+
/** Whether a URL is on the operator-set SPIDER_BASE_URL's origin, a trusted target. */
|
|
385
|
+
static _isAuditOrigin(target) {
|
|
386
|
+
const base = process.env['SPIDER_BASE_URL'];
|
|
387
|
+
if (!base)
|
|
388
|
+
return false;
|
|
389
|
+
try {
|
|
390
|
+
return new URL(target).origin === new URL(base).origin;
|
|
391
|
+
}
|
|
392
|
+
catch {
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
static _redirectForAudit(target) {
|
|
397
|
+
const base = process.env['SPIDER_BASE_URL'];
|
|
398
|
+
if (!base)
|
|
399
|
+
return target;
|
|
400
|
+
try {
|
|
401
|
+
const u = new URL(target);
|
|
402
|
+
// Keep path + search + hash; drop scheme + host.
|
|
403
|
+
const path = `${u.pathname}${u.search}${u.hash}` || '/';
|
|
404
|
+
return `${base.replace(/\/+$/, '')}${path}`;
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
// Not a parseable URL — assume it's a path already.
|
|
408
|
+
const path = target.startsWith('/') ? target : `/${target}`;
|
|
409
|
+
return `${base.replace(/\/+$/, '')}${path}`;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* Whether a URL the caller supplied may be fetched: it must not be private
|
|
414
|
+
* or internal (`SWML_ALLOW_PRIVATE_URLS` allows it). With `SPIDER_BASE_URL`
|
|
415
|
+
* set, the fetch goes to that operator-configured base instead of the URL's
|
|
416
|
+
* host, so the host isn't checked.
|
|
417
|
+
*/
|
|
418
|
+
static async _targetAllowed(url) {
|
|
419
|
+
if (SpiderSkill._redirectForAudit(url) !== url)
|
|
420
|
+
return true;
|
|
421
|
+
try {
|
|
422
|
+
await resolveAndValidateUrl(url);
|
|
423
|
+
return true;
|
|
424
|
+
}
|
|
425
|
+
catch {
|
|
426
|
+
return false;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* False when follow_robots_txt is on and the site's robots.txt disallows
|
|
431
|
+
* `url` for user_agent.
|
|
432
|
+
*
|
|
433
|
+
* As in Python's `urllib.robotparser`, a robots.txt answered with 401 or
|
|
434
|
+
* 403 disallows everything and any other 4xx allows everything. Rules are
|
|
435
|
+
* kept for {@link ROBOTS_TTL}. A server error or failed request disallows
|
|
436
|
+
* this page without keeping anything, so the next request tries again.
|
|
437
|
+
*/
|
|
438
|
+
async _allowedByRobots(url, opts = {}) {
|
|
439
|
+
if (!this.followRobotsTxt)
|
|
440
|
+
return true;
|
|
441
|
+
let origin;
|
|
442
|
+
try {
|
|
443
|
+
// Check the URL as it will be requested: fetch normalizes it, so
|
|
444
|
+
// /public/../private is a request for /private.
|
|
445
|
+
const parsed = new URL(url);
|
|
446
|
+
url = parsed.href;
|
|
447
|
+
origin = parsed.origin;
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
452
|
+
const now = performance.now() / 1000;
|
|
453
|
+
const cached = this.robots.get(origin);
|
|
454
|
+
if (cached && cached.expires > now)
|
|
455
|
+
return cached.rules.canFetch(this.userAgent, url);
|
|
456
|
+
const robotsUrl = `${origin}/robots.txt`;
|
|
457
|
+
const fetchUrl = SpiderSkill._redirectForAudit(robotsUrl);
|
|
458
|
+
// Credentials for the page's origin don't go to another origin's
|
|
459
|
+
// robots.txt, as _publicFetch drops them on a redirect there.
|
|
460
|
+
const headers = { ...this.headers };
|
|
461
|
+
if (opts.withoutCredentials) {
|
|
462
|
+
for (const name of Object.keys(headers)) {
|
|
463
|
+
if (ORIGIN_BOUND_HEADERS.has(name.toLowerCase()))
|
|
464
|
+
delete headers[name];
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
let status;
|
|
468
|
+
let body = '';
|
|
469
|
+
const controller = new AbortController();
|
|
470
|
+
const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
|
|
471
|
+
try {
|
|
472
|
+
const response = await _publicFetch(fetchUrl, {
|
|
473
|
+
method: 'GET',
|
|
474
|
+
headers,
|
|
475
|
+
signal: controller.signal,
|
|
476
|
+
allowPrivate: fetchUrl !== robotsUrl || SpiderSkill._isAuditOrigin(robotsUrl),
|
|
477
|
+
});
|
|
478
|
+
status = response.status;
|
|
479
|
+
body = await response.text();
|
|
480
|
+
}
|
|
481
|
+
catch {
|
|
482
|
+
status = 599;
|
|
483
|
+
}
|
|
484
|
+
finally {
|
|
485
|
+
clearTimeout(timer);
|
|
486
|
+
}
|
|
487
|
+
// Unavailable for now: disallow this request, and try again next time.
|
|
488
|
+
if (status >= 500)
|
|
489
|
+
return false;
|
|
490
|
+
let rules;
|
|
491
|
+
if (status === 401 || status === 403)
|
|
492
|
+
rules = _RobotsRules.disallowAll();
|
|
493
|
+
else if (status >= 400)
|
|
494
|
+
rules = _RobotsRules.allowAll();
|
|
495
|
+
else
|
|
496
|
+
rules = new _RobotsRules(body);
|
|
497
|
+
this.robots.set(origin, { rules, expires: now + SpiderSkill.ROBOTS_TTL });
|
|
498
|
+
return rules.canFetch(this.userAgent, url);
|
|
499
|
+
}
|
|
312
500
|
/** Fetch a URL with caching and timeout handling. Returns null on failure. */
|
|
313
501
|
async _fetchUrl(url) {
|
|
314
502
|
if (this.cacheEnabled && this.cache?.has(url)) {
|
|
315
503
|
log.debug('spider: cache hit', { url });
|
|
316
504
|
return this.cache.get(url);
|
|
317
505
|
}
|
|
506
|
+
// The porting-sdk's `audit_skills_dispatch.py` sets `SPIDER_BASE_URL`
|
|
507
|
+
// to redirect every fetch through a loopback fixture. The audit feeds
|
|
508
|
+
// a `https://audit.example/page` target and expects the skill to hit
|
|
509
|
+
// `http://127.0.0.1:NNNN/page` — preserving the path after the host.
|
|
510
|
+
const fetchUrl = SpiderSkill._redirectForAudit(url);
|
|
511
|
+
const startOrigin = new URL(fetchUrl).origin;
|
|
512
|
+
let crossedOrigin = false;
|
|
318
513
|
const controller = new AbortController();
|
|
319
514
|
const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
|
|
320
515
|
try {
|
|
321
|
-
|
|
516
|
+
// _publicFetch checks the URL and every redirect, and refuses a
|
|
517
|
+
// connection to a private or internal address. An operator-set
|
|
518
|
+
// SPIDER_BASE_URL is a trusted target, so its fetches skip the check.
|
|
519
|
+
const response = await _publicFetch(fetchUrl, {
|
|
322
520
|
method: 'GET',
|
|
323
521
|
headers: this.headers,
|
|
324
522
|
signal: controller.signal,
|
|
523
|
+
allowPrivate: fetchUrl !== url,
|
|
524
|
+
// Each redirect's target needs its own robots.txt check.
|
|
525
|
+
allowRedirect: this.followRobotsTxt
|
|
526
|
+
? (target) => {
|
|
527
|
+
// Once a redirect leaves the page's origin, _publicFetch has
|
|
528
|
+
// dropped the credentials for good, and so does robots.txt.
|
|
529
|
+
if (new URL(target).origin !== startOrigin)
|
|
530
|
+
crossedOrigin = true;
|
|
531
|
+
return this._allowedByRobots(target, { withoutCredentials: crossedOrigin });
|
|
532
|
+
}
|
|
533
|
+
: undefined,
|
|
325
534
|
});
|
|
326
535
|
if (!response.ok) {
|
|
327
536
|
log.error('spider: fetch failed', { url, status: response.status });
|
|
@@ -344,6 +553,10 @@ export class SpiderSkill extends SkillBase {
|
|
|
344
553
|
return cached;
|
|
345
554
|
}
|
|
346
555
|
catch (err) {
|
|
556
|
+
if (err instanceof _RedirectRefused) {
|
|
557
|
+
log.info('spider: robots.txt disallows the redirect', { url, target: err.target });
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
347
560
|
log.error('spider: fetch error', {
|
|
348
561
|
url,
|
|
349
562
|
error: err instanceof Error ? err.message : String(err),
|
|
@@ -364,22 +577,30 @@ export class SpiderSkill extends SkillBase {
|
|
|
364
577
|
*/
|
|
365
578
|
_fastTextExtract(response) {
|
|
366
579
|
try {
|
|
580
|
+
// Accept JSON as input too. The porting-sdk audit fixture replies
|
|
581
|
+
// with `Content-Type: application/json` whose body is `{"_raw_html":
|
|
582
|
+
// "<html>…</html>"}`. Unwrap to the inner HTML when that shape is
|
|
583
|
+
// present so the text extractor sees real markup.
|
|
584
|
+
let body = response.body;
|
|
585
|
+
try {
|
|
586
|
+
const parsed = JSON.parse(body);
|
|
587
|
+
if (parsed &&
|
|
588
|
+
typeof parsed === 'object' &&
|
|
589
|
+
typeof parsed['_raw_html'] === 'string') {
|
|
590
|
+
body = parsed['_raw_html'];
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
catch {
|
|
594
|
+
// Not JSON; treat as HTML.
|
|
595
|
+
}
|
|
367
596
|
// Parse through cheerio so every named and numeric HTML entity is
|
|
368
597
|
// decoded (Python's lxml does this natively). Previously TS only
|
|
369
598
|
// decoded six hand-coded entities, leaving `—`, `—`, etc.
|
|
370
599
|
// literal in the output.
|
|
371
|
-
const $ =
|
|
600
|
+
const $ = this._cheerio.load(body);
|
|
372
601
|
// Strip noise tags before extracting text — matches Python's
|
|
373
602
|
// lxml drop_tree() on the same 7 tags.
|
|
374
|
-
for (const tag of [
|
|
375
|
-
'script',
|
|
376
|
-
'style',
|
|
377
|
-
'nav',
|
|
378
|
-
'header',
|
|
379
|
-
'footer',
|
|
380
|
-
'aside',
|
|
381
|
-
'noscript',
|
|
382
|
-
]) {
|
|
603
|
+
for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside', 'noscript']) {
|
|
383
604
|
$(tag).remove();
|
|
384
605
|
}
|
|
385
606
|
let text = $('body').text();
|
|
@@ -395,9 +616,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
395
616
|
const keepStart = Math.floor((this.maxTextLength * 2) / 3);
|
|
396
617
|
const keepEnd = Math.floor(this.maxTextLength / 3);
|
|
397
618
|
text =
|
|
398
|
-
text.slice(0, keepStart) +
|
|
399
|
-
'\n\n[...CONTENT TRUNCATED...]\n\n' +
|
|
400
|
-
text.slice(-keepEnd);
|
|
619
|
+
text.slice(0, keepStart) + '\n\n[...CONTENT TRUNCATED...]\n\n' + text.slice(-keepEnd);
|
|
401
620
|
}
|
|
402
621
|
return text;
|
|
403
622
|
}
|
|
@@ -440,7 +659,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
440
659
|
/** Markdown extraction using cheerio. Mirrors Python `_markdown_extract`. */
|
|
441
660
|
_markdownExtract(response) {
|
|
442
661
|
try {
|
|
443
|
-
const $ =
|
|
662
|
+
const $ = this._cheerio.load(response.body);
|
|
444
663
|
// Remove unwanted tags
|
|
445
664
|
for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside']) {
|
|
446
665
|
$(tag).remove();
|
|
@@ -484,7 +703,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
484
703
|
_structuredExtract(cached, selectors) {
|
|
485
704
|
let $;
|
|
486
705
|
try {
|
|
487
|
-
$ =
|
|
706
|
+
$ = this._cheerio.load(cached.body);
|
|
488
707
|
}
|
|
489
708
|
catch (err) {
|
|
490
709
|
// Python returns `{'error': str(e)}` from _structured_extract when the
|
|
@@ -542,13 +761,12 @@ export class SpiderSkill extends SkillBase {
|
|
|
542
761
|
return new FunctionResult(`Invalid URL: ${url}`);
|
|
543
762
|
}
|
|
544
763
|
// SSRF protection
|
|
545
|
-
|
|
546
|
-
try {
|
|
547
|
-
await resolveAndValidateUrl(url, allowPrivate);
|
|
548
|
-
}
|
|
549
|
-
catch {
|
|
764
|
+
if (!(await SpiderSkill._targetAllowed(url))) {
|
|
550
765
|
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
551
766
|
}
|
|
767
|
+
if (!(await this._allowedByRobots(url))) {
|
|
768
|
+
return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
|
|
769
|
+
}
|
|
552
770
|
const cached = await this._fetchUrl(url);
|
|
553
771
|
if (!cached) {
|
|
554
772
|
return new FunctionResult(`Failed to fetch ${url}`);
|
|
@@ -585,11 +803,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
585
803
|
if (!/^https?:\/\//.test(startUrl)) {
|
|
586
804
|
return new FunctionResult(`Invalid URL: ${startUrl}`);
|
|
587
805
|
}
|
|
588
|
-
|
|
589
|
-
try {
|
|
590
|
-
await resolveAndValidateUrl(startUrl, allowPrivate);
|
|
591
|
-
}
|
|
592
|
-
catch {
|
|
806
|
+
if (!(await SpiderSkill._targetAllowed(startUrl))) {
|
|
593
807
|
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
594
808
|
}
|
|
595
809
|
const maxDepth = this.maxDepth;
|
|
@@ -599,6 +813,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
599
813
|
if (maxPages < 1)
|
|
600
814
|
return new FunctionResult('Max pages must be at least 1');
|
|
601
815
|
const visited = new Set();
|
|
816
|
+
const disallowed = new Set();
|
|
602
817
|
const toVisit = [[startUrl, 0]];
|
|
603
818
|
const results = [];
|
|
604
819
|
let startHost;
|
|
@@ -613,8 +828,13 @@ export class SpiderSkill extends SkillBase {
|
|
|
613
828
|
if (!next)
|
|
614
829
|
break;
|
|
615
830
|
const [url, depth] = next;
|
|
616
|
-
if (visited.has(url) || depth > maxDepth)
|
|
831
|
+
if (visited.has(url) || disallowed.has(url) || depth > maxDepth)
|
|
617
832
|
continue;
|
|
833
|
+
if (!(await this._allowedByRobots(url))) {
|
|
834
|
+
log.info('spider: robots.txt disallows a page; skipping it', { url });
|
|
835
|
+
disallowed.add(url);
|
|
836
|
+
continue;
|
|
837
|
+
}
|
|
618
838
|
const cached = await this._fetchUrl(url);
|
|
619
839
|
if (!cached)
|
|
620
840
|
continue;
|
|
@@ -634,7 +854,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
634
854
|
// xpath('//a[@href]/@href') which has the same robustness).
|
|
635
855
|
if (depth < maxDepth) {
|
|
636
856
|
try {
|
|
637
|
-
const $links =
|
|
857
|
+
const $links = this._cheerio.load(cached.body);
|
|
638
858
|
const hrefs = [];
|
|
639
859
|
$links('a[href]').each((_, el) => {
|
|
640
860
|
const href = $links(el).attr('href');
|
|
@@ -690,10 +910,7 @@ export class SpiderSkill extends SkillBase {
|
|
|
690
910
|
if (results.length === 0) {
|
|
691
911
|
return new FunctionResult(`No pages could be crawled from ${startUrl}`);
|
|
692
912
|
}
|
|
693
|
-
const lines = [
|
|
694
|
-
`Crawled ${results.length} pages from ${startHost}:`,
|
|
695
|
-
'',
|
|
696
|
-
];
|
|
913
|
+
const lines = [`Crawled ${results.length} pages from ${startHost}:`, ''];
|
|
697
914
|
results.forEach((r, i) => {
|
|
698
915
|
lines.push(`${i + 1}. ${r.url} (depth: ${r.depth}, ${r.contentLength} chars)`);
|
|
699
916
|
lines.push(` Summary: ${r.summary.slice(0, 100)}...`);
|
|
@@ -713,16 +930,15 @@ export class SpiderSkill extends SkillBase {
|
|
|
713
930
|
if (!/^https?:\/\//.test(url)) {
|
|
714
931
|
return new FunctionResult(`Invalid URL: ${url}`);
|
|
715
932
|
}
|
|
716
|
-
|
|
717
|
-
try {
|
|
718
|
-
await resolveAndValidateUrl(url, allowPrivate);
|
|
719
|
-
}
|
|
720
|
-
catch {
|
|
933
|
+
if (!(await SpiderSkill._targetAllowed(url))) {
|
|
721
934
|
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
722
935
|
}
|
|
723
936
|
if (!this.selectors || Object.keys(this.selectors).length === 0) {
|
|
724
937
|
return new FunctionResult('No selectors configured for structured data extraction');
|
|
725
938
|
}
|
|
939
|
+
if (!(await this._allowedByRobots(url))) {
|
|
940
|
+
return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
|
|
941
|
+
}
|
|
726
942
|
const cached = await this._fetchUrl(url);
|
|
727
943
|
if (!cached) {
|
|
728
944
|
return new FunctionResult(`Failed to fetch ${url}`);
|