@signalwire/sdk 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AgentBase.d.ts +1045 -0
- package/dist/AgentBase.js +2375 -0
- package/dist/AgentBase.js.map +1 -0
- package/dist/AgentServer.d.ts +146 -0
- package/dist/AgentServer.js +388 -0
- package/dist/AgentServer.js.map +1 -0
- package/dist/AuthHandler.d.ts +127 -0
- package/dist/AuthHandler.js +212 -0
- package/dist/AuthHandler.js.map +1 -0
- package/dist/ConfigLoader.d.ts +170 -0
- package/dist/ConfigLoader.js +393 -0
- package/dist/ConfigLoader.js.map +1 -0
- package/dist/ContextBuilder.d.ts +652 -0
- package/dist/ContextBuilder.js +1115 -0
- package/dist/ContextBuilder.js.map +1 -0
- package/dist/DataMap.d.ts +271 -0
- package/dist/DataMap.js +430 -0
- package/dist/DataMap.js.map +1 -0
- package/dist/FunctionResult.d.ts +481 -0
- package/dist/FunctionResult.js +775 -0
- package/dist/FunctionResult.js.map +1 -0
- package/dist/Logger.d.ts +113 -0
- package/dist/Logger.js +320 -0
- package/dist/Logger.js.map +1 -0
- package/dist/PomBuilder.d.ts +208 -0
- package/dist/PomBuilder.js +409 -0
- package/dist/PomBuilder.js.map +1 -0
- package/dist/PromptManager.d.ts +85 -0
- package/dist/PromptManager.js +108 -0
- package/dist/PromptManager.js.map +1 -0
- package/dist/SWMLHandler.d.ts +159 -0
- package/dist/SWMLHandler.js +229 -0
- package/dist/SWMLHandler.js.map +1 -0
- package/dist/SWMLService.d.ts +326 -0
- package/dist/SWMLService.js +610 -0
- package/dist/SWMLService.js.map +1 -0
- package/dist/SchemaUtils.d.ts +108 -0
- package/dist/SchemaUtils.js +317 -0
- package/dist/SchemaUtils.js.map +1 -0
- package/dist/SecurityUtils.d.ts +60 -0
- package/{src/SecurityUtils.ts → dist/SecurityUtils.js} +82 -83
- package/dist/SecurityUtils.js.map +1 -0
- package/dist/ServerlessAdapter.d.ts +119 -0
- package/dist/ServerlessAdapter.js +188 -0
- package/dist/ServerlessAdapter.js.map +1 -0
- package/dist/SessionManager.d.ts +142 -0
- package/dist/SessionManager.js +260 -0
- package/dist/SessionManager.js.map +1 -0
- package/dist/SslConfig.d.ts +77 -0
- package/dist/SslConfig.js +100 -0
- package/dist/SslConfig.js.map +1 -0
- package/dist/SwaigFunction.d.ts +159 -0
- package/dist/SwaigFunction.js +209 -0
- package/dist/SwaigFunction.js.map +1 -0
- package/dist/SwmlBuilder.d.ts +165 -0
- package/dist/SwmlBuilder.js +247 -0
- package/dist/SwmlBuilder.js.map +1 -0
- package/dist/SwmlVerbMethods.generated.d.ts +254 -0
- package/dist/SwmlVerbMethods.generated.js +9 -0
- package/dist/SwmlVerbMethods.generated.js.map +1 -0
- package/dist/TypeInference.d.ts +71 -0
- package/dist/TypeInference.js +164 -0
- package/dist/TypeInference.js.map +1 -0
- package/dist/WebService.d.ts +148 -0
- package/dist/WebService.js +532 -0
- package/dist/WebService.js.map +1 -0
- package/dist/cli/agent-loader.d.ts +22 -0
- package/dist/cli/agent-loader.js +144 -0
- package/dist/cli/agent-loader.js.map +1 -0
- package/dist/cli/mock-data.d.ts +37 -0
- package/dist/cli/mock-data.js +80 -0
- package/dist/cli/mock-data.js.map +1 -0
- package/dist/cli/swaig-test.d.ts +32 -0
- package/dist/cli/swaig-test.js +418 -0
- package/dist/cli/swaig-test.js.map +1 -0
- package/dist/generateVerbTypes.d.ts +11 -0
- package/dist/generateVerbTypes.js +265 -0
- package/dist/generateVerbTypes.js.map +1 -0
- package/{src/index.ts → dist/index.d.ts} +13 -100
- package/dist/index.js +152 -0
- package/dist/index.js.map +1 -0
- package/dist/livewire/index.d.ts +653 -0
- package/dist/livewire/index.js +992 -0
- package/dist/livewire/index.js.map +1 -0
- package/dist/prefabs/ConciergeAgent.d.ts +89 -0
- package/dist/prefabs/ConciergeAgent.js +245 -0
- package/dist/prefabs/ConciergeAgent.js.map +1 -0
- package/dist/prefabs/FAQBotAgent.d.ts +113 -0
- package/dist/prefabs/FAQBotAgent.js +329 -0
- package/dist/prefabs/FAQBotAgent.js.map +1 -0
- package/dist/prefabs/InfoGathererAgent.d.ts +126 -0
- package/dist/prefabs/InfoGathererAgent.js +287 -0
- package/dist/prefabs/InfoGathererAgent.js.map +1 -0
- package/dist/prefabs/ReceptionistAgent.d.ts +104 -0
- package/dist/prefabs/ReceptionistAgent.js +303 -0
- package/dist/prefabs/ReceptionistAgent.js.map +1 -0
- package/dist/prefabs/SurveyAgent.d.ts +148 -0
- package/dist/prefabs/SurveyAgent.js +540 -0
- package/dist/prefabs/SurveyAgent.js.map +1 -0
- package/{src/prefabs/index.ts → dist/prefabs/index.d.ts} +1 -15
- package/dist/prefabs/index.js +14 -0
- package/dist/prefabs/index.js.map +1 -0
- package/dist/relay/Action.d.ts +258 -0
- package/dist/relay/Action.js +402 -0
- package/dist/relay/Action.js.map +1 -0
- package/dist/relay/Call.d.ts +697 -0
- package/dist/relay/Call.js +1062 -0
- package/dist/relay/Call.js.map +1 -0
- package/dist/relay/Deferred.d.ts +44 -0
- package/dist/relay/Deferred.js +69 -0
- package/dist/relay/Deferred.js.map +1 -0
- package/dist/relay/Message.d.ts +103 -0
- package/dist/relay/Message.js +169 -0
- package/dist/relay/Message.js.map +1 -0
- package/dist/relay/RelayClient.d.ts +302 -0
- package/dist/relay/RelayClient.js +1026 -0
- package/dist/relay/RelayClient.js.map +1 -0
- package/dist/relay/RelayError.d.ts +15 -0
- package/dist/relay/RelayError.js +20 -0
- package/dist/relay/RelayError.js.map +1 -0
- package/dist/relay/RelayEvent.d.ts +254 -0
- package/dist/relay/RelayEvent.js +509 -0
- package/dist/relay/RelayEvent.js.map +1 -0
- package/dist/relay/constants.d.ts +90 -0
- package/{src/relay/constants.ts → dist/relay/constants.js} +11 -29
- package/dist/relay/constants.js.map +1 -0
- package/dist/relay/index.d.ts +16 -0
- package/dist/relay/index.js +18 -0
- package/dist/relay/index.js.map +1 -0
- package/dist/relay/types.d.ts +142 -0
- package/dist/relay/types.js +5 -0
- package/dist/relay/types.js.map +1 -0
- package/dist/rest/HttpClient.d.ts +80 -0
- package/dist/rest/HttpClient.js +148 -0
- package/dist/rest/HttpClient.js.map +1 -0
- package/dist/rest/RestError.d.ts +30 -0
- package/dist/rest/RestError.js +39 -0
- package/dist/rest/RestError.js.map +1 -0
- package/dist/rest/base/BaseResource.d.ts +24 -0
- package/dist/rest/base/BaseResource.js +29 -0
- package/dist/rest/base/BaseResource.js.map +1 -0
- package/dist/rest/base/CrudResource.d.ts +68 -0
- package/dist/rest/base/CrudResource.js +82 -0
- package/dist/rest/base/CrudResource.js.map +1 -0
- package/dist/rest/base/CrudWithAddresses.d.ts +22 -0
- package/dist/rest/base/CrudWithAddresses.js +25 -0
- package/dist/rest/base/CrudWithAddresses.js.map +1 -0
- package/dist/rest/index.d.ts +131 -0
- package/{src/rest/index.ts → dist/rest/index.js} +101 -136
- package/dist/rest/index.js.map +1 -0
- package/dist/rest/namespaces/addresses.d.ts +46 -0
- package/dist/rest/namespaces/addresses.js +55 -0
- package/dist/rest/namespaces/addresses.js.map +1 -0
- package/dist/rest/namespaces/calling.d.ts +382 -0
- package/dist/rest/namespaces/calling.js +478 -0
- package/dist/rest/namespaces/calling.js.map +1 -0
- package/dist/rest/namespaces/chat.d.ts +22 -0
- package/dist/rest/namespaces/chat.js +26 -0
- package/dist/rest/namespaces/chat.js.map +1 -0
- package/dist/rest/namespaces/compat.d.ts +585 -0
- package/dist/rest/namespaces/compat.js +735 -0
- package/dist/rest/namespaces/compat.js.map +1 -0
- package/dist/rest/namespaces/datasphere.d.ts +66 -0
- package/dist/rest/namespaces/datasphere.js +77 -0
- package/dist/rest/namespaces/datasphere.js.map +1 -0
- package/dist/rest/namespaces/fabric.d.ts +282 -0
- package/dist/rest/namespaces/fabric.js +373 -0
- package/dist/rest/namespaces/fabric.js.map +1 -0
- package/dist/rest/namespaces/imported-numbers.d.ts +22 -0
- package/dist/rest/namespaces/imported-numbers.js +26 -0
- package/dist/rest/namespaces/imported-numbers.js.map +1 -0
- package/dist/rest/namespaces/logs.d.ts +105 -0
- package/dist/rest/namespaces/logs.js +133 -0
- package/dist/rest/namespaces/logs.js.map +1 -0
- package/dist/rest/namespaces/lookup.d.ts +29 -0
- package/dist/rest/namespaces/lookup.js +32 -0
- package/dist/rest/namespaces/lookup.js.map +1 -0
- package/dist/rest/namespaces/mfa.d.ts +46 -0
- package/dist/rest/namespaces/mfa.js +54 -0
- package/dist/rest/namespaces/mfa.js.map +1 -0
- package/dist/rest/namespaces/number-groups.d.ts +49 -0
- package/dist/rest/namespaces/number-groups.js +58 -0
- package/dist/rest/namespaces/number-groups.js.map +1 -0
- package/dist/rest/namespaces/phone-numbers.d.ts +60 -0
- package/dist/rest/namespaces/phone-numbers.js +67 -0
- package/dist/rest/namespaces/phone-numbers.js.map +1 -0
- package/dist/rest/namespaces/project.d.ts +46 -0
- package/dist/rest/namespaces/project.js +56 -0
- package/dist/rest/namespaces/project.js.map +1 -0
- package/dist/rest/namespaces/pubsub.d.ts +22 -0
- package/dist/rest/namespaces/pubsub.js +26 -0
- package/dist/rest/namespaces/pubsub.js.map +1 -0
- package/dist/rest/namespaces/queues.d.ts +42 -0
- package/dist/rest/namespaces/queues.js +49 -0
- package/dist/rest/namespaces/queues.js.map +1 -0
- package/dist/rest/namespaces/recordings.d.ts +38 -0
- package/dist/rest/namespaces/recordings.js +45 -0
- package/dist/rest/namespaces/recordings.js.map +1 -0
- package/dist/rest/namespaces/registry.d.ts +141 -0
- package/dist/rest/namespaces/registry.js +178 -0
- package/dist/rest/namespaces/registry.js.map +1 -0
- package/dist/rest/namespaces/short-codes.d.ts +39 -0
- package/dist/rest/namespaces/short-codes.js +46 -0
- package/dist/rest/namespaces/short-codes.js.map +1 -0
- package/dist/rest/namespaces/sip-profile.d.ts +28 -0
- package/dist/rest/namespaces/sip-profile.js +34 -0
- package/dist/rest/namespaces/sip-profile.js.map +1 -0
- package/dist/rest/namespaces/verified-callers.d.ts +33 -0
- package/dist/rest/namespaces/verified-callers.js +39 -0
- package/dist/rest/namespaces/verified-callers.js.map +1 -0
- package/dist/rest/namespaces/video.d.ts +235 -0
- package/dist/rest/namespaces/video.js +297 -0
- package/dist/rest/namespaces/video.js.map +1 -0
- package/{src/rest/pagination.ts → dist/rest/pagination.d.ts} +2 -61
- package/dist/rest/pagination.js +78 -0
- package/dist/rest/pagination.js.map +1 -0
- package/dist/rest/types.d.ts +54 -0
- package/dist/rest/types.js +5 -0
- package/dist/rest/types.js.map +1 -0
- package/dist/skills/SkillBase.d.ts +404 -0
- package/dist/skills/SkillBase.js +471 -0
- package/dist/skills/SkillBase.js.map +1 -0
- package/dist/skills/SkillManager.d.ts +195 -0
- package/dist/skills/SkillManager.js +378 -0
- package/dist/skills/SkillManager.js.map +1 -0
- package/dist/skills/SkillRegistry.d.ts +176 -0
- package/dist/skills/SkillRegistry.js +310 -0
- package/dist/skills/SkillRegistry.js.map +1 -0
- package/dist/skills/builtin/api_ninjas_trivia.d.ts +47 -0
- package/dist/skills/builtin/api_ninjas_trivia.js +211 -0
- package/dist/skills/builtin/api_ninjas_trivia.js.map +1 -0
- package/dist/skills/builtin/ask_claude.d.ts +37 -0
- package/dist/skills/builtin/ask_claude.js +162 -0
- package/dist/skills/builtin/ask_claude.js.map +1 -0
- package/dist/skills/builtin/claude_skills.d.ts +75 -0
- package/dist/skills/builtin/claude_skills.js +753 -0
- package/dist/skills/builtin/claude_skills.js.map +1 -0
- package/dist/skills/builtin/custom_skills.d.ts +79 -0
- package/dist/skills/builtin/custom_skills.js +227 -0
- package/dist/skills/builtin/custom_skills.js.map +1 -0
- package/dist/skills/builtin/datasphere.d.ts +73 -0
- package/dist/skills/builtin/datasphere.js +350 -0
- package/dist/skills/builtin/datasphere.js.map +1 -0
- package/dist/skills/builtin/datasphere_serverless.d.ts +89 -0
- package/dist/skills/builtin/datasphere_serverless.js +291 -0
- package/dist/skills/builtin/datasphere_serverless.js.map +1 -0
- package/dist/skills/builtin/datetime.d.ts +40 -0
- package/dist/skills/builtin/datetime.js +118 -0
- package/dist/skills/builtin/datetime.js.map +1 -0
- package/dist/skills/builtin/google_maps.d.ts +48 -0
- package/dist/skills/builtin/google_maps.js +470 -0
- package/dist/skills/builtin/google_maps.js.map +1 -0
- package/dist/skills/builtin/index.d.ts +31 -0
- package/{src/skills/builtin/index.ts → dist/skills/builtin/index.js} +30 -36
- package/dist/skills/builtin/index.js.map +1 -0
- package/dist/skills/builtin/info_gatherer.d.ts +115 -0
- package/dist/skills/builtin/info_gatherer.js +328 -0
- package/dist/skills/builtin/info_gatherer.js.map +1 -0
- package/dist/skills/builtin/joke.d.ts +41 -0
- package/dist/skills/builtin/joke.js +156 -0
- package/dist/skills/builtin/joke.js.map +1 -0
- package/dist/skills/builtin/math.d.ts +39 -0
- package/dist/skills/builtin/math.js +119 -0
- package/dist/skills/builtin/math.js.map +1 -0
- package/dist/skills/builtin/mcp_gateway.d.ts +82 -0
- package/dist/skills/builtin/mcp_gateway.js +574 -0
- package/dist/skills/builtin/mcp_gateway.js.map +1 -0
- package/dist/skills/builtin/native_vector_search.d.ts +128 -0
- package/dist/skills/builtin/native_vector_search.js +752 -0
- package/dist/skills/builtin/native_vector_search.js.map +1 -0
- package/dist/skills/builtin/play_background_file.d.ts +59 -0
- package/dist/skills/builtin/play_background_file.js +274 -0
- package/dist/skills/builtin/play_background_file.js.map +1 -0
- package/dist/skills/builtin/spider.d.ts +91 -0
- package/dist/skills/builtin/spider.js +758 -0
- package/dist/skills/builtin/spider.js.map +1 -0
- package/dist/skills/builtin/swml_transfer.d.ts +66 -0
- package/dist/skills/builtin/swml_transfer.js +475 -0
- package/dist/skills/builtin/swml_transfer.js.map +1 -0
- package/dist/skills/builtin/weather_api.d.ts +59 -0
- package/dist/skills/builtin/weather_api.js +196 -0
- package/dist/skills/builtin/weather_api.js.map +1 -0
- package/dist/skills/builtin/web_search.d.ts +160 -0
- package/dist/skills/builtin/web_search.js +894 -0
- package/dist/skills/builtin/web_search.js.map +1 -0
- package/dist/skills/builtin/wikipedia_search.d.ts +77 -0
- package/dist/skills/builtin/wikipedia_search.js +241 -0
- package/dist/skills/builtin/wikipedia_search.js.map +1 -0
- package/{src/skills/index.ts → dist/skills/index.d.ts} +0 -1
- package/dist/skills/index.js +10 -0
- package/dist/skills/index.js.map +1 -0
- package/dist/types.d.ts +118 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/package.json +7 -1
- package/.github/workflows/publish.yml +0 -27
- package/.github/workflows/test.yml +0 -17
- package/CLAUDE.md +0 -112
- package/docs/MIGRATION-2.0.md +0 -81
- package/docs/agent-guide.md +0 -869
- package/docs/agent_guide.md +0 -1026
- package/docs/api-reference.md +0 -2420
- package/docs/api_reference.md +0 -1301
- package/docs/architecture.md +0 -843
- package/docs/cli-guide.md +0 -579
- package/docs/cli_guide.md +0 -448
- package/docs/cloud_functions_guide.md +0 -474
- package/docs/configuration.md +0 -422
- package/docs/contexts-guide.md +0 -787
- package/docs/contexts_guide.md +0 -511
- package/docs/datamap-guide.md +0 -1141
- package/docs/datamap_guide.md +0 -595
- package/docs/llm_parameters.md +0 -239
- package/docs/mcp_gateway_reference.md +0 -648
- package/docs/mcp_integration.md +0 -108
- package/docs/prefabs-guide.md +0 -800
- package/docs/sdk_features.md +0 -477
- package/docs/security.md +0 -436
- package/docs/serverless-guide.md +0 -513
- package/docs/skills-guide.md +0 -857
- package/docs/skills-system.md +0 -301
- package/docs/skills_parameter_schema.md +0 -392
- package/docs/skills_system.md +0 -467
- package/docs/swaig-reference.md +0 -1627
- package/docs/swaig_reference.md +0 -910
- package/docs/swml_service_guide.md +0 -600
- package/docs/third_party_skills.md +0 -533
- package/docs/web_service.md +0 -677
- package/examples/README.md +0 -131
- package/examples/advanced-datamap.ts +0 -100
- package/examples/advanced-dynamic-config.ts +0 -70
- package/examples/auto-vivified.ts +0 -94
- package/examples/call-flow.ts +0 -63
- package/examples/comprehensive-dynamic.ts +0 -125
- package/examples/contexts-steps.ts +0 -65
- package/examples/custom-path.ts +0 -45
- package/examples/datamap-tools.ts +0 -55
- package/examples/datasphere-multi-instance.ts +0 -63
- package/examples/datasphere-serverless-env.ts +0 -68
- package/examples/datasphere-webhook-env.ts +0 -66
- package/examples/datasphere.ts +0 -37
- package/examples/declarative.ts +0 -100
- package/examples/dynamic-config.ts +0 -54
- package/examples/dynamic-info-gatherer.ts +0 -69
- package/examples/dynamic-swml-service.ts +0 -67
- package/examples/gather-info.ts +0 -71
- package/examples/gather-per-question-functions-demo.ts +0 -140
- package/examples/joke-agent.ts +0 -34
- package/examples/kubernetes-agent.ts +0 -45
- package/examples/llm-params.ts +0 -69
- package/examples/mcp-agent.ts +0 -84
- package/examples/mcp-gateway.ts +0 -35
- package/examples/multi-agent.ts +0 -73
- package/examples/multi-endpoint.ts +0 -79
- package/examples/pom-prompt.ts +0 -71
- package/examples/prefab-concierge.ts +0 -47
- package/examples/prefab-faq.ts +0 -54
- package/examples/prefab-info-gatherer.ts +0 -33
- package/examples/prefab-receptionist.ts +0 -36
- package/examples/prefab-survey.ts +0 -68
- package/examples/record-call.ts +0 -67
- package/examples/relay-demo.ts +0 -33
- package/examples/room-and-sip.ts +0 -94
- package/examples/serverless-lambda.ts +0 -35
- package/examples/session-state.ts +0 -83
- package/examples/simple-agent.ts +0 -36
- package/examples/simple-static.ts +0 -29
- package/examples/skills-demo.ts +0 -38
- package/examples/step-function-inheritance-demo.ts +0 -124
- package/examples/swaig-features.ts +0 -187
- package/examples/swml-service-routing.ts +0 -66
- package/examples/swml-service.ts +0 -38
- package/examples/tap.ts +0 -68
- package/examples/verb-methods.ts +0 -45
- package/examples/web-search-multi-instance.ts +0 -72
- package/examples/web-search.ts +0 -38
- package/examples/wikipedia.ts +0 -36
- package/livewire/README.md +0 -140
- package/livewire/docs/migration-guide.md +0 -228
- package/livewire/examples/livewire-basic-agent.ts +0 -74
- package/livewire/examples/livewire-handoff.ts +0 -197
- package/livewire/examples/livewire-multi-tool.ts +0 -117
- package/relay/README.md +0 -70
- package/relay/RELAY_IMPLEMENTATION_GUIDE.md +0 -1257
- package/relay/docs/call-methods.md +0 -474
- package/relay/docs/client-reference.md +0 -142
- package/relay/docs/events.md +0 -134
- package/relay/docs/getting-started.md +0 -134
- package/relay/docs/guide.md +0 -306
- package/relay/docs/messaging.md +0 -182
- package/relay/examples/relay-inbound.ts +0 -66
- package/relay/examples/relay-messaging.ts +0 -71
- package/relay/examples/relay-outbound.ts +0 -76
- package/rest/README.md +0 -95
- package/rest/docs/calling.md +0 -273
- package/rest/docs/client-reference.md +0 -107
- package/rest/docs/compat.md +0 -204
- package/rest/docs/fabric.md +0 -183
- package/rest/docs/getting-started.md +0 -110
- package/rest/docs/guide.md +0 -351
- package/rest/docs/namespaces.md +0 -285
- package/rest/examples/rest-10dlc-registration.ts +0 -161
- package/rest/examples/rest-calling-ivr-and-ai.ts +0 -102
- package/rest/examples/rest-calling-play-and-record.ts +0 -91
- package/rest/examples/rest-client.ts +0 -65
- package/rest/examples/rest-compat-laml.ts +0 -210
- package/rest/examples/rest-datasphere-search.ts +0 -82
- package/rest/examples/rest-fabric-conferences-and-routing.ts +0 -144
- package/rest/examples/rest-fabric-subscribers-and-sip.ts +0 -108
- package/rest/examples/rest-fabric-swml-and-callflows.ts +0 -99
- package/rest/examples/rest-manage-resources.ts +0 -65
- package/rest/examples/rest-phone-number-management.ts +0 -197
- package/rest/examples/rest-queues-mfa-and-recordings.ts +0 -143
- package/rest/examples/rest-video-rooms.ts +0 -186
- package/src/AgentBase.ts +0 -2598
- package/src/AgentServer.ts +0 -442
- package/src/AuthHandler.ts +0 -256
- package/src/ConfigLoader.ts +0 -447
- package/src/ContextBuilder.ts +0 -1181
- package/src/DataMap.ts +0 -477
- package/src/FunctionResult.ts +0 -903
- package/src/Logger.ts +0 -337
- package/src/PomBuilder.ts +0 -467
- package/src/PromptManager.ts +0 -133
- package/src/SWMLHandler.ts +0 -298
- package/src/SWMLService.ts +0 -749
- package/src/SchemaUtils.ts +0 -361
- package/src/ServerlessAdapter.ts +0 -240
- package/src/SessionManager.ts +0 -311
- package/src/SslConfig.ts +0 -118
- package/src/SwaigFunction.ts +0 -276
- package/src/SwmlBuilder.ts +0 -283
- package/src/SwmlVerbMethods.generated.ts +0 -234
- package/src/TypeInference.ts +0 -199
- package/src/WebService.ts +0 -641
- package/src/cli/agent-loader.ts +0 -166
- package/src/cli/mock-data.ts +0 -111
- package/src/cli/swaig-test.ts +0 -452
- package/src/generateVerbTypes.ts +0 -322
- package/src/livewire/index.ts +0 -1252
- package/src/prefabs/ConciergeAgent.ts +0 -309
- package/src/prefabs/FAQBotAgent.ts +0 -415
- package/src/prefabs/InfoGathererAgent.ts +0 -379
- package/src/prefabs/ReceptionistAgent.ts +0 -395
- package/src/prefabs/SurveyAgent.ts +0 -671
- package/src/relay/Action.ts +0 -482
- package/src/relay/Call.ts +0 -1270
- package/src/relay/Deferred.ts +0 -94
- package/src/relay/Message.ts +0 -199
- package/src/relay/RelayClient.ts +0 -1185
- package/src/relay/RelayError.ts +0 -20
- package/src/relay/RelayEvent.ts +0 -944
- package/src/relay/index.ts +0 -123
- package/src/relay/types.ts +0 -149
- package/src/rest/HttpClient.ts +0 -164
- package/src/rest/RestError.ts +0 -45
- package/src/rest/base/BaseResource.ts +0 -32
- package/src/rest/base/CrudResource.ts +0 -98
- package/src/rest/base/CrudWithAddresses.ts +0 -37
- package/src/rest/namespaces/addresses.ts +0 -64
- package/src/rest/namespaces/calling.ts +0 -534
- package/src/rest/namespaces/chat.ts +0 -31
- package/src/rest/namespaces/compat.ts +0 -812
- package/src/rest/namespaces/datasphere.ts +0 -88
- package/src/rest/namespaces/fabric.ts +0 -417
- package/src/rest/namespaces/imported-numbers.ts +0 -31
- package/src/rest/namespaces/logs.ts +0 -151
- package/src/rest/namespaces/lookup.ts +0 -38
- package/src/rest/namespaces/mfa.ts +0 -61
- package/src/rest/namespaces/number-groups.ts +0 -68
- package/src/rest/namespaces/phone-numbers.ts +0 -76
- package/src/rest/namespaces/project.ts +0 -65
- package/src/rest/namespaces/pubsub.ts +0 -31
- package/src/rest/namespaces/queues.ts +0 -58
- package/src/rest/namespaces/recordings.ts +0 -53
- package/src/rest/namespaces/registry.ts +0 -200
- package/src/rest/namespaces/short-codes.ts +0 -54
- package/src/rest/namespaces/sip-profile.ts +0 -40
- package/src/rest/namespaces/verified-callers.ts +0 -46
- package/src/rest/namespaces/video.ts +0 -332
- package/src/rest/types.ts +0 -61
- package/src/skills/SkillBase.ts +0 -582
- package/src/skills/SkillManager.ts +0 -419
- package/src/skills/SkillRegistry.ts +0 -367
- package/src/skills/builtin/api_ninjas_trivia.ts +0 -285
- package/src/skills/builtin/ask_claude.ts +0 -246
- package/src/skills/builtin/claude_skills.ts +0 -983
- package/src/skills/builtin/custom_skills.ts +0 -327
- package/src/skills/builtin/datasphere.ts +0 -447
- package/src/skills/builtin/datasphere_serverless.ts +0 -347
- package/src/skills/builtin/datetime.ts +0 -137
- package/src/skills/builtin/google_maps.ts +0 -656
- package/src/skills/builtin/info_gatherer.ts +0 -415
- package/src/skills/builtin/joke.ts +0 -196
- package/src/skills/builtin/math.ts +0 -149
- package/src/skills/builtin/mcp_gateway.ts +0 -685
- package/src/skills/builtin/native_vector_search.ts +0 -910
- package/src/skills/builtin/play_background_file.ts +0 -367
- package/src/skills/builtin/spider.ts +0 -874
- package/src/skills/builtin/swml_transfer.ts +0 -645
- package/src/skills/builtin/weather_api.ts +0 -255
- package/src/skills/builtin/web_search.ts +0 -1083
- package/src/skills/builtin/wikipedia_search.ts +0 -304
- package/src/types.ts +0 -132
- package/tests/AgentBase.test.ts +0 -1093
- package/tests/AgentServer.test.ts +0 -126
- package/tests/AuthHandler.test.ts +0 -133
- package/tests/ConfigLoader.test.ts +0 -151
- package/tests/ContextBuilder.test.ts +0 -108
- package/tests/DataMap.test.ts +0 -367
- package/tests/EphemeralSkillCopy.test.ts +0 -107
- package/tests/FunctionResult.test.ts +0 -298
- package/tests/GracefulShutdown.test.ts +0 -46
- package/tests/Logger.test.ts +0 -420
- package/tests/PomBuilder.test.ts +0 -228
- package/tests/ProxyDetection.test.ts +0 -228
- package/tests/SWMLService.test.ts +0 -320
- package/tests/SWMLVerbHandler.test.ts +0 -523
- package/tests/SchemaUtils.test.ts +0 -106
- package/tests/SchemaUtils.verb.test.ts +0 -206
- package/tests/SecurityUtils.test.ts +0 -192
- package/tests/ServerlessAdapter.test.ts +0 -222
- package/tests/SessionManager.test.ts +0 -257
- package/tests/SkillBaseFeatures.test.ts +0 -130
- package/tests/SkillMultiInstance.test.ts +0 -102
- package/tests/SkillParameterSchemas.test.ts +0 -134
- package/tests/SkillRegistrySchema.test.ts +0 -84
- package/tests/SslConfig.test.ts +0 -97
- package/tests/SwmlBuilder.test.ts +0 -400
- package/tests/TypeInference.test.ts +0 -165
- package/tests/VerbHandlerRegistry.test.ts +0 -128
- package/tests/cli/mock-data.test.ts +0 -113
- package/tests/cli/swaig-test.test.ts +0 -108
- package/tests/examples.test.ts +0 -194
- package/tests/fixtures/scrape-parity/README.md +0 -53
- package/tests/fixtures/scrape-parity/expected/01_simple_article.json +0 -5
- package/tests/fixtures/scrape-parity/expected/02_class_based_noise.json +0 -5
- package/tests/fixtures/scrape-parity/expected/03_nested_containers.json +0 -5
- package/tests/fixtures/scrape-parity/expected/04_bare_body.json +0 -5
- package/tests/fixtures/scrape-parity/expected/05_malformed.json +0 -5
- package/tests/fixtures/scrape-parity/expected/06_multi_class.json +0 -5
- package/tests/fixtures/scrape-parity/expected/07_anchor_pattern.json +0 -5
- package/tests/fixtures/scrape-parity/expected/08_inline_script.json +0 -5
- package/tests/fixtures/scrape-parity/expected/09_whitespace_nbsp.json +0 -5
- package/tests/fixtures/scrape-parity/expected/10_entity_unicode.json +0 -5
- package/tests/fixtures/scrape-parity/expected/11_cdata_comments.json +0 -5
- package/tests/fixtures/scrape-parity/expected/12_deep_nested_removals.json +0 -5
- package/tests/fixtures/scrape-parity/html/01_simple_article.html +0 -18
- package/tests/fixtures/scrape-parity/html/02_class_based_noise.html +0 -18
- package/tests/fixtures/scrape-parity/html/03_nested_containers.html +0 -13
- package/tests/fixtures/scrape-parity/html/04_bare_body.html +0 -9
- package/tests/fixtures/scrape-parity/html/05_malformed.html +0 -11
- package/tests/fixtures/scrape-parity/html/06_multi_class.html +0 -18
- package/tests/fixtures/scrape-parity/html/07_anchor_pattern.html +0 -12
- package/tests/fixtures/scrape-parity/html/08_inline_script.html +0 -17
- package/tests/fixtures/scrape-parity/html/09_whitespace_nbsp.html +0 -21
- package/tests/fixtures/scrape-parity/html/10_entity_unicode.html +0 -10
- package/tests/fixtures/scrape-parity/html/11_cdata_comments.html +0 -13
- package/tests/fixtures/scrape-parity/html/12_deep_nested_removals.html +0 -24
- package/tests/fixtures/scrape-parity/python_extract.py +0 -109
- package/tests/livewire/livewire.test.ts +0 -780
- package/tests/mcp.test.ts +0 -221
- package/tests/prefabs/prefabs.test.ts +0 -1093
- package/tests/relay/Action.test.ts +0 -392
- package/tests/relay/Call.test.ts +0 -535
- package/tests/relay/Deferred.test.ts +0 -71
- package/tests/relay/Message.test.ts +0 -164
- package/tests/relay/RelayClient.test.ts +0 -779
- package/tests/relay/RelayEvent.test.ts +0 -463
- package/tests/relay/helpers.ts +0 -79
- package/tests/rest/HttpClient.test.ts +0 -274
- package/tests/rest/RestError.test.ts +0 -53
- package/tests/rest/calling.test.ts +0 -156
- package/tests/rest/client.test.ts +0 -197
- package/tests/rest/compat.test.ts +0 -335
- package/tests/rest/datasphere.test.ts +0 -71
- package/tests/rest/fabric.test.ts +0 -186
- package/tests/rest/helpers.ts +0 -95
- package/tests/rest/logs.test.ts +0 -68
- package/tests/rest/pagination.test.ts +0 -133
- package/tests/rest/registry.test.ts +0 -94
- package/tests/rest/relay-rest.test.ts +0 -290
- package/tests/rest/video.test.ts +0 -173
- package/tests/setup.ts +0 -6
- package/tests/skills/SkillBase.test.ts +0 -108
- package/tests/skills/SkillManager.test.ts +0 -268
- package/tests/skills/SkillRegistry.test.ts +0 -176
- package/tests/skills/builtin.test.ts +0 -1472
- package/tests/skills/claude-skills.test.ts +0 -54
- package/tests/skills/custom-skills.test.ts +0 -53
- package/tests/skills/datasphere-serverless.test.ts +0 -70
- package/tests/skills/datasphere.test.ts +0 -107
- package/tests/skills/datetime.test.ts +0 -101
- package/tests/skills/google-maps.test.ts +0 -78
- package/tests/skills/info-gatherer.test.ts +0 -411
- package/tests/skills/joke.test.ts +0 -81
- package/tests/skills/math.test.ts +0 -68
- package/tests/skills/mcp-gateway.test.ts +0 -101
- package/tests/skills/play-background.test.ts +0 -54
- package/tests/skills/spider.test.ts +0 -108
- package/tests/skills/swml-transfer.test.ts +0 -178
- package/tests/skills/trivia.test.ts +0 -54
- package/tests/skills/vector-search.test.ts +0 -135
- package/tests/skills/weather-api.test.ts +0 -77
- package/tests/skills/web-search-parity.test.ts +0 -59
- package/tests/skills/web-search.test.ts +0 -103
- package/tests/skills/wikipedia.test.ts +0 -63
- package/tsconfig.json +0 -18
- package/vitest.config.ts +0 -9
- /package/{src → dist}/schema.json +0 -0
|
@@ -0,0 +1,758 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spider Skill - Fast web scraping and crawling capabilities.
|
|
3
|
+
*
|
|
4
|
+
* Tier 2 built-in skill. Port of the Python `SpiderSkill` that performs
|
|
5
|
+
* HTTP scraping via `fetch`, lightweight HTML-to-text extraction, and
|
|
6
|
+
* breadth-first crawling with configurable limits. Supports three tools:
|
|
7
|
+
*
|
|
8
|
+
* - `scrape_url` — single-page text/markdown extraction
|
|
9
|
+
* - `crawl_site` — breadth-first crawl from a start URL
|
|
10
|
+
* - `extract_structured_data` — CSS/XPath-like structured extraction
|
|
11
|
+
*
|
|
12
|
+
* Multiple instances are supported; use `tool_name` in config to differentiate.
|
|
13
|
+
*/
|
|
14
|
+
import { SkillBase } from '../SkillBase.js';
|
|
15
|
+
import { FunctionResult } from '../../FunctionResult.js';
|
|
16
|
+
import { resolveAndValidateUrl, validateUrl, MAX_SKILL_INPUT_LENGTH } from '../../SecurityUtils.js';
|
|
17
|
+
import { getLogger } from '../../Logger.js';
|
|
18
|
+
import * as cheerio from 'cheerio';
|
|
19
|
+
const log = getLogger('SpiderSkill');
|
|
20
|
+
const WHITESPACE_REGEX = /\s+/g;
|
|
21
|
+
/** Tags stripped from HTML before text extraction. */
|
|
22
|
+
const STRIP_TAG_REGEXES = [
|
|
23
|
+
/<script\b[^>]*>[\s\S]*?<\/script>/gi,
|
|
24
|
+
/<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
|
25
|
+
/<nav\b[^>]*>[\s\S]*?<\/nav>/gi,
|
|
26
|
+
/<header\b[^>]*>[\s\S]*?<\/header>/gi,
|
|
27
|
+
/<footer\b[^>]*>[\s\S]*?<\/footer>/gi,
|
|
28
|
+
/<aside\b[^>]*>[\s\S]*?<\/aside>/gi,
|
|
29
|
+
/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi,
|
|
30
|
+
];
|
|
31
|
+
/**
|
|
32
|
+
* Fast web scraping skill optimized for speed and token efficiency.
|
|
33
|
+
*
|
|
34
|
+
* Multi-instance capable. Port of the Python `SpiderSkill` with three tools:
|
|
35
|
+
* `scrape_url`, `crawl_site`, and `extract_structured_data`. Configuration
|
|
36
|
+
* mirrors the Python schema (delay, concurrent_requests, timeout, max_pages,
|
|
37
|
+
* max_depth, extract_type, max_text_length, clean_text, selectors,
|
|
38
|
+
* follow_patterns, user_agent, headers, follow_robots_txt, cache_enabled).
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```ts
|
|
42
|
+
* agent.addSkill('spider', { max_pages: 5, max_depth: 2 });
|
|
43
|
+
* ```
|
|
44
|
+
*/
|
|
45
|
+
export class SpiderSkill extends SkillBase {
|
|
46
|
+
// Python ground truth: skills/spider/skill.py:~140-145
|
|
47
|
+
// REQUIRED_PACKAGES = ["lxml"] in Python; TS uses cheerio.
|
|
48
|
+
static SKILL_NAME = 'spider';
|
|
49
|
+
static SKILL_DESCRIPTION = 'Fast web scraping and crawling capabilities';
|
|
50
|
+
static SKILL_VERSION = '1.0.0';
|
|
51
|
+
static REQUIRED_PACKAGES = ['cheerio'];
|
|
52
|
+
static REQUIRED_ENV_VARS = [];
|
|
53
|
+
static SUPPORTS_MULTIPLE_INSTANCES = true;
|
|
54
|
+
static getParameterSchema() {
|
|
55
|
+
return {
|
|
56
|
+
...super.getParameterSchema(),
|
|
57
|
+
delay: {
|
|
58
|
+
type: 'number',
|
|
59
|
+
description: 'Delay between requests in seconds',
|
|
60
|
+
default: 0.1,
|
|
61
|
+
required: false,
|
|
62
|
+
min: 0,
|
|
63
|
+
},
|
|
64
|
+
concurrent_requests: {
|
|
65
|
+
type: 'integer',
|
|
66
|
+
description: 'Number of concurrent requests allowed',
|
|
67
|
+
default: 5,
|
|
68
|
+
required: false,
|
|
69
|
+
min: 1,
|
|
70
|
+
max: 20,
|
|
71
|
+
},
|
|
72
|
+
timeout: {
|
|
73
|
+
type: 'integer',
|
|
74
|
+
description: 'Request timeout in seconds',
|
|
75
|
+
default: 5,
|
|
76
|
+
required: false,
|
|
77
|
+
min: 1,
|
|
78
|
+
max: 60,
|
|
79
|
+
},
|
|
80
|
+
max_pages: {
|
|
81
|
+
type: 'integer',
|
|
82
|
+
description: 'Maximum number of pages to scrape',
|
|
83
|
+
default: 1,
|
|
84
|
+
required: false,
|
|
85
|
+
min: 1,
|
|
86
|
+
max: 100,
|
|
87
|
+
},
|
|
88
|
+
max_depth: {
|
|
89
|
+
type: 'integer',
|
|
90
|
+
description: 'Maximum crawl depth (0 = single page only)',
|
|
91
|
+
default: 0,
|
|
92
|
+
required: false,
|
|
93
|
+
min: 0,
|
|
94
|
+
max: 5,
|
|
95
|
+
},
|
|
96
|
+
extract_type: {
|
|
97
|
+
type: 'string',
|
|
98
|
+
description: 'Content extraction method',
|
|
99
|
+
default: 'fast_text',
|
|
100
|
+
required: false,
|
|
101
|
+
enum: ['fast_text', 'clean_text', 'full_text', 'html', 'markdown', 'structured', 'custom'],
|
|
102
|
+
},
|
|
103
|
+
max_text_length: {
|
|
104
|
+
type: 'integer',
|
|
105
|
+
description: 'Maximum text length to return',
|
|
106
|
+
default: 3000,
|
|
107
|
+
required: false,
|
|
108
|
+
min: 100,
|
|
109
|
+
max: 100000,
|
|
110
|
+
},
|
|
111
|
+
clean_text: {
|
|
112
|
+
type: 'boolean',
|
|
113
|
+
description: 'Whether to clean extracted text',
|
|
114
|
+
default: true,
|
|
115
|
+
required: false,
|
|
116
|
+
},
|
|
117
|
+
selectors: {
|
|
118
|
+
type: 'object',
|
|
119
|
+
description: 'Custom CSS/XPath selectors for structured extraction',
|
|
120
|
+
default: {},
|
|
121
|
+
required: false,
|
|
122
|
+
additionalProperties: { type: 'string' },
|
|
123
|
+
},
|
|
124
|
+
follow_patterns: {
|
|
125
|
+
type: 'array',
|
|
126
|
+
description: 'URL patterns to follow when crawling',
|
|
127
|
+
default: [],
|
|
128
|
+
required: false,
|
|
129
|
+
items: { type: 'string' },
|
|
130
|
+
},
|
|
131
|
+
user_agent: {
|
|
132
|
+
type: 'string',
|
|
133
|
+
description: 'User agent string for requests',
|
|
134
|
+
default: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
135
|
+
required: false,
|
|
136
|
+
},
|
|
137
|
+
headers: {
|
|
138
|
+
type: 'object',
|
|
139
|
+
description: 'Additional HTTP headers',
|
|
140
|
+
default: {},
|
|
141
|
+
required: false,
|
|
142
|
+
additionalProperties: { type: 'string' },
|
|
143
|
+
},
|
|
144
|
+
follow_robots_txt: {
|
|
145
|
+
type: 'boolean',
|
|
146
|
+
// Matches Python's __init__ runtime fallback (skills/spider/skill.py:177
|
|
147
|
+
// uses `params.get('follow_robots_txt', False)`). Python's schema said
|
|
148
|
+
// True but the runtime and TS both use False — aligning the schema
|
|
149
|
+
// eliminates the cross-SDK contract mismatch.
|
|
150
|
+
description: 'Whether to respect robots.txt (default: false to match Python runtime)',
|
|
151
|
+
default: false,
|
|
152
|
+
required: false,
|
|
153
|
+
},
|
|
154
|
+
cache_enabled: {
|
|
155
|
+
type: 'boolean',
|
|
156
|
+
description: 'Whether to cache scraped pages',
|
|
157
|
+
default: true,
|
|
158
|
+
required: false,
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
// Runtime state, populated in setup()
|
|
163
|
+
delay = 0.1;
|
|
164
|
+
concurrentRequests = 5;
|
|
165
|
+
timeout = 5;
|
|
166
|
+
maxPages = 1;
|
|
167
|
+
maxDepth = 0;
|
|
168
|
+
extractType = 'fast_text';
|
|
169
|
+
maxTextLength = 3000;
|
|
170
|
+
cleanText = true;
|
|
171
|
+
cacheEnabled = true;
|
|
172
|
+
followRobotsTxt = true;
|
|
173
|
+
userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
|
|
174
|
+
headers = {};
|
|
175
|
+
selectors = {};
|
|
176
|
+
compiledFollowPatterns = [];
|
|
177
|
+
cache = null;
|
|
178
|
+
cacheMaxSize = 100;
|
|
179
|
+
getInstanceKey() {
|
|
180
|
+
const toolName = this.getConfig('tool_name', this.skillName);
|
|
181
|
+
return `${this.skillName}_${toolName}`;
|
|
182
|
+
}
|
|
183
|
+
async setup() {
|
|
184
|
+
// Performance
|
|
185
|
+
this.delay = this.getConfig('delay', 0.1);
|
|
186
|
+
this.concurrentRequests = this.getConfig('concurrent_requests', 5);
|
|
187
|
+
this.timeout = this.getConfig('timeout', 5);
|
|
188
|
+
// Crawl limits
|
|
189
|
+
this.maxPages = this.getConfig('max_pages', 1);
|
|
190
|
+
this.maxDepth = this.getConfig('max_depth', 0);
|
|
191
|
+
// Content processing
|
|
192
|
+
this.extractType = this.getConfig('extract_type', 'fast_text');
|
|
193
|
+
// Python has an internal inconsistency (schema: 10000, __init__ fallback:
|
|
194
|
+
// 3000). The effective runtime default is 3000 because Python reads via
|
|
195
|
+
// `self.params.get('max_text_length', 3000)` and schema defaults are not
|
|
196
|
+
// applied. Use 3000 for parity. Follow-up: reconcile Python schema vs init.
|
|
197
|
+
this.maxTextLength = this.getConfig('max_text_length', 3000);
|
|
198
|
+
this.cleanText = this.getConfig('clean_text', true);
|
|
199
|
+
// Features
|
|
200
|
+
this.cacheEnabled = this.getConfig('cache_enabled', true);
|
|
201
|
+
this.followRobotsTxt = this.getConfig('follow_robots_txt', false);
|
|
202
|
+
this.userAgent = this.getConfig('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
|
|
203
|
+
// Optional headers + user-agent merge
|
|
204
|
+
const headers = this.getConfig('headers', {}) ?? {};
|
|
205
|
+
this.headers = { ...headers, 'User-Agent': this.userAgent };
|
|
206
|
+
// Selectors for structured extraction
|
|
207
|
+
this.selectors = this.getConfig('selectors', {}) ?? {};
|
|
208
|
+
// Setup cache
|
|
209
|
+
this.cache = this.cacheEnabled ? new Map() : null;
|
|
210
|
+
// Validate numeric ranges
|
|
211
|
+
if (this.delay < 0) {
|
|
212
|
+
log.error('spider: delay cannot be negative', { delay: this.delay });
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
if (this.concurrentRequests < 1 || this.concurrentRequests > 20) {
|
|
216
|
+
log.error('spider: concurrent_requests must be between 1 and 20', { concurrent_requests: this.concurrentRequests });
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
219
|
+
if (this.maxPages < 1) {
|
|
220
|
+
log.error('spider: max_pages must be at least 1', { max_pages: this.maxPages });
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
if (this.maxDepth < 0) {
|
|
224
|
+
log.error('spider: max_depth cannot be negative', { max_depth: this.maxDepth });
|
|
225
|
+
return false;
|
|
226
|
+
}
|
|
227
|
+
// Pre-compile follow patterns
|
|
228
|
+
this.compiledFollowPatterns = [];
|
|
229
|
+
const patterns = this.getConfig('follow_patterns', []) ?? [];
|
|
230
|
+
for (const pattern of patterns) {
|
|
231
|
+
try {
|
|
232
|
+
this.compiledFollowPatterns.push(new RegExp(pattern));
|
|
233
|
+
}
|
|
234
|
+
catch (err) {
|
|
235
|
+
log.error('spider: invalid follow pattern', {
|
|
236
|
+
pattern,
|
|
237
|
+
error: err instanceof Error ? err.message : String(err),
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
log.info('spider: configured', {
|
|
242
|
+
delay: this.delay,
|
|
243
|
+
max_pages: this.maxPages,
|
|
244
|
+
max_depth: this.maxDepth,
|
|
245
|
+
});
|
|
246
|
+
return true;
|
|
247
|
+
}
|
|
248
|
+
getHints() {
|
|
249
|
+
return [
|
|
250
|
+
'scrape',
|
|
251
|
+
'crawl',
|
|
252
|
+
'extract',
|
|
253
|
+
'web page',
|
|
254
|
+
'website',
|
|
255
|
+
'get content from',
|
|
256
|
+
'fetch data from',
|
|
257
|
+
'spider',
|
|
258
|
+
];
|
|
259
|
+
}
|
|
260
|
+
async cleanup() {
|
|
261
|
+
if (this.cache) {
|
|
262
|
+
this.cache.clear();
|
|
263
|
+
}
|
|
264
|
+
log.info('spider: cleaned up');
|
|
265
|
+
}
|
|
266
|
+
/** @returns Three tools: `scrape_url`, `crawl_site`, and `extract_structured_data`. */
|
|
267
|
+
getTools() {
|
|
268
|
+
const toolPrefix = this.getConfig('tool_name', '');
|
|
269
|
+
const prefix = toolPrefix ? `${toolPrefix}_` : '';
|
|
270
|
+
return [
|
|
271
|
+
{
|
|
272
|
+
name: `${prefix}scrape_url`,
|
|
273
|
+
description: 'Extract text content from a single web page',
|
|
274
|
+
parameters: {
|
|
275
|
+
url: {
|
|
276
|
+
type: 'string',
|
|
277
|
+
description: 'The URL to scrape',
|
|
278
|
+
},
|
|
279
|
+
},
|
|
280
|
+
required: ['url'],
|
|
281
|
+
handler: async (args, rawData) => this._scrapeUrlHandler(args, rawData),
|
|
282
|
+
},
|
|
283
|
+
{
|
|
284
|
+
name: `${prefix}crawl_site`,
|
|
285
|
+
description: 'Crawl multiple pages starting from a URL',
|
|
286
|
+
parameters: {
|
|
287
|
+
start_url: {
|
|
288
|
+
type: 'string',
|
|
289
|
+
description: 'Starting URL for the crawl',
|
|
290
|
+
},
|
|
291
|
+
},
|
|
292
|
+
required: ['start_url'],
|
|
293
|
+
handler: async (args, rawData) => this._crawlSiteHandler(args, rawData),
|
|
294
|
+
},
|
|
295
|
+
{
|
|
296
|
+
name: `${prefix}extract_structured_data`,
|
|
297
|
+
description: 'Extract specific data from a web page using selectors',
|
|
298
|
+
parameters: {
|
|
299
|
+
url: {
|
|
300
|
+
type: 'string',
|
|
301
|
+
description: 'The URL to scrape',
|
|
302
|
+
},
|
|
303
|
+
},
|
|
304
|
+
required: ['url'],
|
|
305
|
+
handler: async (args, rawData) => this._extractStructuredHandler(args, rawData),
|
|
306
|
+
},
|
|
307
|
+
];
|
|
308
|
+
}
|
|
309
|
+
// ---------------------------------------------------------------------------
|
|
310
|
+
// Internal helpers
|
|
311
|
+
// ---------------------------------------------------------------------------
|
|
312
|
+
/** Fetch a URL with caching and timeout handling. Returns null on failure. */
|
|
313
|
+
async _fetchUrl(url) {
|
|
314
|
+
if (this.cacheEnabled && this.cache?.has(url)) {
|
|
315
|
+
log.debug('spider: cache hit', { url });
|
|
316
|
+
return this.cache.get(url);
|
|
317
|
+
}
|
|
318
|
+
const controller = new AbortController();
|
|
319
|
+
const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
|
|
320
|
+
try {
|
|
321
|
+
const response = await fetch(url, {
|
|
322
|
+
method: 'GET',
|
|
323
|
+
headers: this.headers,
|
|
324
|
+
signal: controller.signal,
|
|
325
|
+
});
|
|
326
|
+
if (!response.ok) {
|
|
327
|
+
log.error('spider: fetch failed', { url, status: response.status });
|
|
328
|
+
return null;
|
|
329
|
+
}
|
|
330
|
+
const body = await response.text();
|
|
331
|
+
const cached = {
|
|
332
|
+
url: response.url || url,
|
|
333
|
+
status: response.status,
|
|
334
|
+
body,
|
|
335
|
+
};
|
|
336
|
+
if (this.cacheEnabled && this.cache) {
|
|
337
|
+
if (this.cache.size >= this.cacheMaxSize) {
|
|
338
|
+
const oldest = this.cache.keys().next().value;
|
|
339
|
+
if (oldest !== undefined)
|
|
340
|
+
this.cache.delete(oldest);
|
|
341
|
+
}
|
|
342
|
+
this.cache.set(url, cached);
|
|
343
|
+
}
|
|
344
|
+
return cached;
|
|
345
|
+
}
|
|
346
|
+
catch (err) {
|
|
347
|
+
log.error('spider: fetch error', {
|
|
348
|
+
url,
|
|
349
|
+
error: err instanceof Error ? err.message : String(err),
|
|
350
|
+
});
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
finally {
|
|
354
|
+
clearTimeout(timer);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Extract plain text from a fetched response using simple regex stripping.
|
|
359
|
+
*
|
|
360
|
+
* Takes a {@link CachedResponse} (TS equivalent of `requests.Response`) to
|
|
361
|
+
* match Python's `_fast_text_extract(response)` capability surface — the
|
|
362
|
+
* response-like object exposes `url`, `status`, and `body` for callers that
|
|
363
|
+
* need to branch on status/content-type.
|
|
364
|
+
*/
|
|
365
|
+
_fastTextExtract(response) {
|
|
366
|
+
try {
|
|
367
|
+
// Parse through cheerio so every named and numeric HTML entity is
|
|
368
|
+
// decoded (Python's lxml does this natively). Previously TS only
|
|
369
|
+
// decoded six hand-coded entities, leaving `—`, `—`, etc.
|
|
370
|
+
// literal in the output.
|
|
371
|
+
const $ = cheerio.load(response.body);
|
|
372
|
+
// Strip noise tags before extracting text — matches Python's
|
|
373
|
+
// lxml drop_tree() on the same 7 tags.
|
|
374
|
+
for (const tag of [
|
|
375
|
+
'script',
|
|
376
|
+
'style',
|
|
377
|
+
'nav',
|
|
378
|
+
'header',
|
|
379
|
+
'footer',
|
|
380
|
+
'aside',
|
|
381
|
+
'noscript',
|
|
382
|
+
]) {
|
|
383
|
+
$(tag).remove();
|
|
384
|
+
}
|
|
385
|
+
let text = $('body').text();
|
|
386
|
+
if (!text || text.trim().length === 0) {
|
|
387
|
+
// Pages without <body> fall through to document-root text.
|
|
388
|
+
text = $.root().text();
|
|
389
|
+
}
|
|
390
|
+
if (this.cleanText) {
|
|
391
|
+
text = text.replace(WHITESPACE_REGEX, ' ').trim();
|
|
392
|
+
}
|
|
393
|
+
// Smart truncation
|
|
394
|
+
if (text.length > this.maxTextLength) {
|
|
395
|
+
const keepStart = Math.floor((this.maxTextLength * 2) / 3);
|
|
396
|
+
const keepEnd = Math.floor(this.maxTextLength / 3);
|
|
397
|
+
text =
|
|
398
|
+
text.slice(0, keepStart) +
|
|
399
|
+
'\n\n[...CONTENT TRUNCATED...]\n\n' +
|
|
400
|
+
text.slice(-keepEnd);
|
|
401
|
+
}
|
|
402
|
+
return text;
|
|
403
|
+
}
|
|
404
|
+
catch (err) {
|
|
405
|
+
// Python swallows extraction errors and returns '' so the handler
|
|
406
|
+
// emits a clean "no content extracted" message instead of bubbling
|
|
407
|
+
// a parse failure all the way up to the AI.
|
|
408
|
+
log.error('spider: fast-text extraction failed', {
|
|
409
|
+
error: err instanceof Error ? err.message : String(err),
|
|
410
|
+
});
|
|
411
|
+
return '';
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Extract a value using a CSS selector (via cheerio) or XPath-like
|
|
416
|
+
* selector (`//tag`). Mirrors Python `_structured_extract`'s selector
|
|
417
|
+
* branching: selectors starting with `/` are treated as XPath (tag-name
|
|
418
|
+
* subset), everything else is full CSS via cheerio's querySelector-style
|
|
419
|
+
* engine.
|
|
420
|
+
*/
|
|
421
|
+
_applySelector($, selector) {
|
|
422
|
+
const trimmed = selector.trim();
|
|
423
|
+
if (!trimmed)
|
|
424
|
+
return [];
|
|
425
|
+
if (trimmed.startsWith('/')) {
|
|
426
|
+
// Subset of XPath supported in Python's lxml path — `//tag`, `//tag/text()`.
|
|
427
|
+
const tagMatch = /\/\/([a-zA-Z][a-zA-Z0-9]*)/.exec(trimmed);
|
|
428
|
+
if (!tagMatch)
|
|
429
|
+
return [];
|
|
430
|
+
return $(tagMatch[1])
|
|
431
|
+
.toArray()
|
|
432
|
+
.map((el) => $(el).text().trim())
|
|
433
|
+
.filter((t) => t.length > 0);
|
|
434
|
+
}
|
|
435
|
+
return $(trimmed)
|
|
436
|
+
.toArray()
|
|
437
|
+
.map((el) => $(el).text().trim())
|
|
438
|
+
.filter((t) => t.length > 0);
|
|
439
|
+
}
|
|
440
|
+
/** Markdown extraction using cheerio. Mirrors Python `_markdown_extract`. */
|
|
441
|
+
_markdownExtract(response) {
|
|
442
|
+
try {
|
|
443
|
+
const $ = cheerio.load(response.body);
|
|
444
|
+
// Remove unwanted tags
|
|
445
|
+
for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside']) {
|
|
446
|
+
$(tag).remove();
|
|
447
|
+
}
|
|
448
|
+
const parts = [];
|
|
449
|
+
const title = $('title').first().text().trim();
|
|
450
|
+
if (title)
|
|
451
|
+
parts.push(`# ${title}\n`);
|
|
452
|
+
$('h1, h2, h3, h4, h5, h6, p, li, code, pre').each((_, el) => {
|
|
453
|
+
const tag = el.tagName.toLowerCase();
|
|
454
|
+
const text = $(el).text().trim();
|
|
455
|
+
if (!text)
|
|
456
|
+
return;
|
|
457
|
+
if (/^h[1-6]$/.test(tag)) {
|
|
458
|
+
const level = Number(tag[1]);
|
|
459
|
+
parts.push(`\n${'#'.repeat(level)} ${text}\n`);
|
|
460
|
+
}
|
|
461
|
+
else if (tag === 'p') {
|
|
462
|
+
parts.push(`\n${text}\n`);
|
|
463
|
+
}
|
|
464
|
+
else if (tag === 'li') {
|
|
465
|
+
parts.push(`- ${text}`);
|
|
466
|
+
}
|
|
467
|
+
else if (tag === 'code' || tag === 'pre') {
|
|
468
|
+
parts.push(`\n\`\`\`\n${text}\n\`\`\`\n`);
|
|
469
|
+
}
|
|
470
|
+
});
|
|
471
|
+
let text = parts.join('\n');
|
|
472
|
+
if (text.length > this.maxTextLength) {
|
|
473
|
+
text = text.slice(0, this.maxTextLength) + '\n\n[...TRUNCATED...]';
|
|
474
|
+
}
|
|
475
|
+
return text;
|
|
476
|
+
}
|
|
477
|
+
catch (err) {
|
|
478
|
+
log.error('spider: markdown extraction error', {
|
|
479
|
+
error: err instanceof Error ? err.message : String(err),
|
|
480
|
+
});
|
|
481
|
+
return this._fastTextExtract(response);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
_structuredExtract(cached, selectors) {
|
|
485
|
+
let $;
|
|
486
|
+
try {
|
|
487
|
+
$ = cheerio.load(cached.body);
|
|
488
|
+
}
|
|
489
|
+
catch (err) {
|
|
490
|
+
// Python returns `{'error': str(e)}` from _structured_extract when the
|
|
491
|
+
// parser fails (skill.py). Surface the same shape so upstream handlers
|
|
492
|
+
// can branch on `'error' in result`.
|
|
493
|
+
log.error('spider: structured parse failed', {
|
|
494
|
+
error: err instanceof Error ? err.message : String(err),
|
|
495
|
+
});
|
|
496
|
+
return { error: err instanceof Error ? err.message : String(err) };
|
|
497
|
+
}
|
|
498
|
+
const result = {
|
|
499
|
+
url: cached.url,
|
|
500
|
+
status_code: cached.status,
|
|
501
|
+
title: $('title').first().text().trim(),
|
|
502
|
+
data: {},
|
|
503
|
+
};
|
|
504
|
+
const data = {};
|
|
505
|
+
for (const [field, selector] of Object.entries(selectors)) {
|
|
506
|
+
try {
|
|
507
|
+
const values = this._applySelector($, selector);
|
|
508
|
+
if (values.length === 0) {
|
|
509
|
+
data[field] = null;
|
|
510
|
+
}
|
|
511
|
+
else if (values.length === 1) {
|
|
512
|
+
data[field] = values[0];
|
|
513
|
+
}
|
|
514
|
+
else {
|
|
515
|
+
data[field] = values;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
catch (err) {
|
|
519
|
+
log.warn('spider: selector error', {
|
|
520
|
+
selector,
|
|
521
|
+
error: err instanceof Error ? err.message : String(err),
|
|
522
|
+
});
|
|
523
|
+
data[field] = null;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
result['data'] = data;
|
|
527
|
+
return result;
|
|
528
|
+
}
|
|
529
|
+
// ---------------------------------------------------------------------------
|
|
530
|
+
// Tool handlers
|
|
531
|
+
// ---------------------------------------------------------------------------
|
|
532
|
+
async _scrapeUrlHandler(args, _rawData) {
|
|
533
|
+
const urlArg = args['url'];
|
|
534
|
+
if (typeof urlArg !== 'string' || urlArg.trim().length === 0) {
|
|
535
|
+
return new FunctionResult('Please provide a URL to scrape');
|
|
536
|
+
}
|
|
537
|
+
const url = urlArg.trim();
|
|
538
|
+
if (url.length > MAX_SKILL_INPUT_LENGTH) {
|
|
539
|
+
return new FunctionResult('Input URL is too long.');
|
|
540
|
+
}
|
|
541
|
+
if (!/^https?:\/\//.test(url)) {
|
|
542
|
+
return new FunctionResult(`Invalid URL: ${url}`);
|
|
543
|
+
}
|
|
544
|
+
// SSRF protection
|
|
545
|
+
const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
|
|
546
|
+
try {
|
|
547
|
+
await resolveAndValidateUrl(url, allowPrivate);
|
|
548
|
+
}
|
|
549
|
+
catch {
|
|
550
|
+
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
551
|
+
}
|
|
552
|
+
const cached = await this._fetchUrl(url);
|
|
553
|
+
if (!cached) {
|
|
554
|
+
return new FunctionResult(`Failed to fetch ${url}`);
|
|
555
|
+
}
|
|
556
|
+
try {
|
|
557
|
+
if (this.extractType === 'structured') {
|
|
558
|
+
const result = this._structuredExtract(cached, this.selectors);
|
|
559
|
+
return new FunctionResult(`Extracted structured data from ${url}: ${JSON.stringify(result)}`);
|
|
560
|
+
}
|
|
561
|
+
const content = this.extractType === 'markdown'
|
|
562
|
+
? this._markdownExtract(cached)
|
|
563
|
+
: this._fastTextExtract(cached);
|
|
564
|
+
if (!content) {
|
|
565
|
+
return new FunctionResult(`No content extracted from ${url}`);
|
|
566
|
+
}
|
|
567
|
+
const charCount = content.length;
|
|
568
|
+
const header = `Content from ${url} (${charCount} characters):\n\n`;
|
|
569
|
+
return new FunctionResult(header + content);
|
|
570
|
+
}
|
|
571
|
+
catch (err) {
|
|
572
|
+
log.error('spider: scrape_url error', {
|
|
573
|
+
url,
|
|
574
|
+
error: err instanceof Error ? err.message : String(err),
|
|
575
|
+
});
|
|
576
|
+
return new FunctionResult(`Error processing ${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
async _crawlSiteHandler(args, _rawData) {
|
|
580
|
+
const startArg = args['start_url'];
|
|
581
|
+
if (typeof startArg !== 'string' || startArg.trim().length === 0) {
|
|
582
|
+
return new FunctionResult('Please provide a starting URL for the crawl');
|
|
583
|
+
}
|
|
584
|
+
const startUrl = startArg.trim();
|
|
585
|
+
if (!/^https?:\/\//.test(startUrl)) {
|
|
586
|
+
return new FunctionResult(`Invalid URL: ${startUrl}`);
|
|
587
|
+
}
|
|
588
|
+
const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
|
|
589
|
+
try {
|
|
590
|
+
await resolveAndValidateUrl(startUrl, allowPrivate);
|
|
591
|
+
}
|
|
592
|
+
catch {
|
|
593
|
+
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
594
|
+
}
|
|
595
|
+
const maxDepth = this.maxDepth;
|
|
596
|
+
const maxPages = this.maxPages;
|
|
597
|
+
if (maxDepth < 0)
|
|
598
|
+
return new FunctionResult('Max depth cannot be negative');
|
|
599
|
+
if (maxPages < 1)
|
|
600
|
+
return new FunctionResult('Max pages must be at least 1');
|
|
601
|
+
const visited = new Set();
|
|
602
|
+
const toVisit = [[startUrl, 0]];
|
|
603
|
+
const results = [];
|
|
604
|
+
let startHost;
|
|
605
|
+
try {
|
|
606
|
+
startHost = new URL(startUrl).host;
|
|
607
|
+
}
|
|
608
|
+
catch {
|
|
609
|
+
return new FunctionResult(`Invalid URL: ${startUrl}`);
|
|
610
|
+
}
|
|
611
|
+
while (toVisit.length > 0 && visited.size < maxPages) {
|
|
612
|
+
const next = toVisit.shift();
|
|
613
|
+
if (!next)
|
|
614
|
+
break;
|
|
615
|
+
const [url, depth] = next;
|
|
616
|
+
if (visited.has(url) || depth > maxDepth)
|
|
617
|
+
continue;
|
|
618
|
+
const cached = await this._fetchUrl(url);
|
|
619
|
+
if (!cached)
|
|
620
|
+
continue;
|
|
621
|
+
visited.add(url);
|
|
622
|
+
const content = this._fastTextExtract(cached);
|
|
623
|
+
if (content) {
|
|
624
|
+
results.push({
|
|
625
|
+
url,
|
|
626
|
+
depth,
|
|
627
|
+
contentLength: content.length,
|
|
628
|
+
summary: content.length > 500 ? content.slice(0, 500) + '...' : content,
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
// Extract links if not at max depth. Use cheerio so we correctly
|
|
632
|
+
// pick up unquoted and oddly-whitespaced href attributes that the
|
|
633
|
+
// regex path missed (Python skills/spider/skill.py:487 uses lxml
|
|
634
|
+
// xpath('//a[@href]/@href') which has the same robustness).
|
|
635
|
+
if (depth < maxDepth) {
|
|
636
|
+
try {
|
|
637
|
+
const $links = cheerio.load(cached.body);
|
|
638
|
+
const hrefs = [];
|
|
639
|
+
$links('a[href]').each((_, el) => {
|
|
640
|
+
const href = $links(el).attr('href');
|
|
641
|
+
if (href)
|
|
642
|
+
hrefs.push(href);
|
|
643
|
+
});
|
|
644
|
+
for (const href of hrefs) {
|
|
645
|
+
let absoluteUrl;
|
|
646
|
+
try {
|
|
647
|
+
absoluteUrl = new URL(href, url).toString();
|
|
648
|
+
}
|
|
649
|
+
catch {
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
if (this.compiledFollowPatterns.length > 0) {
|
|
653
|
+
const allowed = this.compiledFollowPatterns.some((re) => re.test(absoluteUrl));
|
|
654
|
+
if (!allowed)
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
657
|
+
try {
|
|
658
|
+
const absHost = new URL(absoluteUrl).host;
|
|
659
|
+
if (absHost !== startHost)
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
catch {
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
// SSRF hardening — discovered links must pass the same
|
|
666
|
+
// validation the entry URL did. Prevents a crafted page from
|
|
667
|
+
// bouncing the crawler into internal/metadata endpoints.
|
|
668
|
+
if (!(await validateUrl(absoluteUrl))) {
|
|
669
|
+
log.debug('spider: skipping discovered URL rejected by SSRF', {
|
|
670
|
+
url: absoluteUrl,
|
|
671
|
+
});
|
|
672
|
+
continue;
|
|
673
|
+
}
|
|
674
|
+
if (!visited.has(absoluteUrl)) {
|
|
675
|
+
toVisit.push([absoluteUrl, depth + 1]);
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
catch (err) {
|
|
680
|
+
log.warn('spider: link extraction error', {
|
|
681
|
+
url,
|
|
682
|
+
error: err instanceof Error ? err.message : String(err),
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
if (this.delay > 0 && visited.size < maxPages) {
|
|
687
|
+
await new Promise((resolve) => setTimeout(resolve, this.delay * 1000));
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
if (results.length === 0) {
|
|
691
|
+
return new FunctionResult(`No pages could be crawled from ${startUrl}`);
|
|
692
|
+
}
|
|
693
|
+
const lines = [
|
|
694
|
+
`Crawled ${results.length} pages from ${startHost}:`,
|
|
695
|
+
'',
|
|
696
|
+
];
|
|
697
|
+
results.forEach((r, i) => {
|
|
698
|
+
lines.push(`${i + 1}. ${r.url} (depth: ${r.depth}, ${r.contentLength} chars)`);
|
|
699
|
+
lines.push(` Summary: ${r.summary.slice(0, 100)}...`);
|
|
700
|
+
lines.push('');
|
|
701
|
+
});
|
|
702
|
+
const totalChars = results.reduce((acc, r) => acc + r.contentLength, 0);
|
|
703
|
+
lines.push('');
|
|
704
|
+
lines.push(`Total content: ${totalChars.toLocaleString()} characters across ${results.length} pages`);
|
|
705
|
+
return new FunctionResult(lines.join('\n'));
|
|
706
|
+
}
|
|
707
|
+
async _extractStructuredHandler(args, _rawData) {
|
|
708
|
+
const urlArg = args['url'];
|
|
709
|
+
if (typeof urlArg !== 'string' || urlArg.trim().length === 0) {
|
|
710
|
+
return new FunctionResult('Please provide a URL');
|
|
711
|
+
}
|
|
712
|
+
const url = urlArg.trim();
|
|
713
|
+
if (!/^https?:\/\//.test(url)) {
|
|
714
|
+
return new FunctionResult(`Invalid URL: ${url}`);
|
|
715
|
+
}
|
|
716
|
+
const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
|
|
717
|
+
try {
|
|
718
|
+
await resolveAndValidateUrl(url, allowPrivate);
|
|
719
|
+
}
|
|
720
|
+
catch {
|
|
721
|
+
return new FunctionResult('URL rejected: cannot access private or internal URLs');
|
|
722
|
+
}
|
|
723
|
+
if (!this.selectors || Object.keys(this.selectors).length === 0) {
|
|
724
|
+
return new FunctionResult('No selectors configured for structured data extraction');
|
|
725
|
+
}
|
|
726
|
+
const cached = await this._fetchUrl(url);
|
|
727
|
+
if (!cached) {
|
|
728
|
+
return new FunctionResult(`Failed to fetch ${url}`);
|
|
729
|
+
}
|
|
730
|
+
const result = this._structuredExtract(cached, this.selectors);
|
|
731
|
+
if ('error' in result) {
|
|
732
|
+
return new FunctionResult(`Error extracting data: ${result['error']}`);
|
|
733
|
+
}
|
|
734
|
+
const lines = [`Extracted data from ${url}:`, ''];
|
|
735
|
+
lines.push(`Title: ${result['title'] || 'N/A'}`);
|
|
736
|
+
lines.push('');
|
|
737
|
+
const data = result['data'];
|
|
738
|
+
if (data && Object.keys(data).length > 0) {
|
|
739
|
+
lines.push('Data:');
|
|
740
|
+
for (const [field, value] of Object.entries(data)) {
|
|
741
|
+
lines.push(`- ${field}: ${String(value)}`);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
else {
|
|
745
|
+
lines.push('No data extracted with provided selectors');
|
|
746
|
+
}
|
|
747
|
+
return new FunctionResult(lines.join('\n'));
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Factory function for creating SpiderSkill instances.
|
|
752
|
+
* @param config - Optional skill configuration.
|
|
753
|
+
* @returns A new SpiderSkill instance.
|
|
754
|
+
*/
|
|
755
|
+
export function createSkill(config) {
|
|
756
|
+
return new SpiderSkill(config);
|
|
757
|
+
}
|
|
758
|
+
//# sourceMappingURL=spider.js.map
|