@signalwire/sdk 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (325) hide show
  1. package/.github/workflows/publish.yml +27 -0
  2. package/.github/workflows/test.yml +17 -0
  3. package/CLAUDE.md +112 -0
  4. package/README.md +254 -0
  5. package/docs/MIGRATION-2.0.md +81 -0
  6. package/docs/agent-guide.md +869 -0
  7. package/docs/agent_guide.md +1026 -0
  8. package/docs/api-reference.md +2420 -0
  9. package/docs/api_reference.md +1301 -0
  10. package/docs/architecture.md +843 -0
  11. package/docs/cli-guide.md +579 -0
  12. package/docs/cli_guide.md +448 -0
  13. package/docs/cloud_functions_guide.md +474 -0
  14. package/docs/configuration.md +422 -0
  15. package/docs/contexts-guide.md +787 -0
  16. package/docs/contexts_guide.md +511 -0
  17. package/docs/datamap-guide.md +1141 -0
  18. package/docs/datamap_guide.md +595 -0
  19. package/docs/llm_parameters.md +239 -0
  20. package/docs/mcp_gateway_reference.md +648 -0
  21. package/docs/mcp_integration.md +108 -0
  22. package/docs/prefabs-guide.md +800 -0
  23. package/docs/sdk_features.md +477 -0
  24. package/docs/security.md +436 -0
  25. package/docs/serverless-guide.md +513 -0
  26. package/docs/skills-guide.md +857 -0
  27. package/docs/skills-system.md +301 -0
  28. package/docs/skills_parameter_schema.md +392 -0
  29. package/docs/skills_system.md +467 -0
  30. package/docs/swaig-reference.md +1627 -0
  31. package/docs/swaig_reference.md +910 -0
  32. package/docs/swml_service_guide.md +600 -0
  33. package/docs/third_party_skills.md +533 -0
  34. package/docs/web_service.md +677 -0
  35. package/examples/README.md +131 -0
  36. package/examples/advanced-datamap.ts +100 -0
  37. package/examples/advanced-dynamic-config.ts +70 -0
  38. package/examples/auto-vivified.ts +94 -0
  39. package/examples/call-flow.ts +63 -0
  40. package/examples/comprehensive-dynamic.ts +125 -0
  41. package/examples/contexts-steps.ts +65 -0
  42. package/examples/custom-path.ts +45 -0
  43. package/examples/datamap-tools.ts +55 -0
  44. package/examples/datasphere-multi-instance.ts +63 -0
  45. package/examples/datasphere-serverless-env.ts +68 -0
  46. package/examples/datasphere-webhook-env.ts +66 -0
  47. package/examples/datasphere.ts +37 -0
  48. package/examples/declarative.ts +100 -0
  49. package/examples/dynamic-config.ts +54 -0
  50. package/examples/dynamic-info-gatherer.ts +69 -0
  51. package/examples/dynamic-swml-service.ts +67 -0
  52. package/examples/gather-info.ts +71 -0
  53. package/examples/gather-per-question-functions-demo.ts +140 -0
  54. package/examples/joke-agent.ts +34 -0
  55. package/examples/kubernetes-agent.ts +45 -0
  56. package/examples/llm-params.ts +69 -0
  57. package/examples/mcp-agent.ts +84 -0
  58. package/examples/mcp-gateway.ts +35 -0
  59. package/examples/multi-agent.ts +73 -0
  60. package/examples/multi-endpoint.ts +79 -0
  61. package/examples/pom-prompt.ts +71 -0
  62. package/examples/prefab-concierge.ts +47 -0
  63. package/examples/prefab-faq.ts +54 -0
  64. package/examples/prefab-info-gatherer.ts +33 -0
  65. package/examples/prefab-receptionist.ts +36 -0
  66. package/examples/prefab-survey.ts +68 -0
  67. package/examples/record-call.ts +67 -0
  68. package/examples/relay-demo.ts +33 -0
  69. package/examples/room-and-sip.ts +94 -0
  70. package/examples/serverless-lambda.ts +35 -0
  71. package/examples/session-state.ts +83 -0
  72. package/examples/simple-agent.ts +36 -0
  73. package/examples/simple-static.ts +29 -0
  74. package/examples/skills-demo.ts +38 -0
  75. package/examples/step-function-inheritance-demo.ts +124 -0
  76. package/examples/swaig-features.ts +187 -0
  77. package/examples/swml-service-routing.ts +66 -0
  78. package/examples/swml-service.ts +38 -0
  79. package/examples/tap.ts +68 -0
  80. package/examples/verb-methods.ts +45 -0
  81. package/examples/web-search-multi-instance.ts +72 -0
  82. package/examples/web-search.ts +38 -0
  83. package/examples/wikipedia.ts +36 -0
  84. package/livewire/README.md +140 -0
  85. package/livewire/docs/migration-guide.md +228 -0
  86. package/livewire/examples/livewire-basic-agent.ts +74 -0
  87. package/livewire/examples/livewire-handoff.ts +197 -0
  88. package/livewire/examples/livewire-multi-tool.ts +117 -0
  89. package/package.json +38 -0
  90. package/relay/README.md +70 -0
  91. package/relay/RELAY_IMPLEMENTATION_GUIDE.md +1257 -0
  92. package/relay/docs/call-methods.md +474 -0
  93. package/relay/docs/client-reference.md +142 -0
  94. package/relay/docs/events.md +134 -0
  95. package/relay/docs/getting-started.md +134 -0
  96. package/relay/docs/guide.md +306 -0
  97. package/relay/docs/messaging.md +182 -0
  98. package/relay/examples/relay-inbound.ts +66 -0
  99. package/relay/examples/relay-messaging.ts +71 -0
  100. package/relay/examples/relay-outbound.ts +76 -0
  101. package/rest/README.md +95 -0
  102. package/rest/docs/calling.md +273 -0
  103. package/rest/docs/client-reference.md +107 -0
  104. package/rest/docs/compat.md +204 -0
  105. package/rest/docs/fabric.md +183 -0
  106. package/rest/docs/getting-started.md +110 -0
  107. package/rest/docs/guide.md +351 -0
  108. package/rest/docs/namespaces.md +285 -0
  109. package/rest/examples/rest-10dlc-registration.ts +161 -0
  110. package/rest/examples/rest-calling-ivr-and-ai.ts +102 -0
  111. package/rest/examples/rest-calling-play-and-record.ts +91 -0
  112. package/rest/examples/rest-client.ts +65 -0
  113. package/rest/examples/rest-compat-laml.ts +210 -0
  114. package/rest/examples/rest-datasphere-search.ts +82 -0
  115. package/rest/examples/rest-fabric-conferences-and-routing.ts +144 -0
  116. package/rest/examples/rest-fabric-subscribers-and-sip.ts +108 -0
  117. package/rest/examples/rest-fabric-swml-and-callflows.ts +99 -0
  118. package/rest/examples/rest-manage-resources.ts +65 -0
  119. package/rest/examples/rest-phone-number-management.ts +197 -0
  120. package/rest/examples/rest-queues-mfa-and-recordings.ts +143 -0
  121. package/rest/examples/rest-video-rooms.ts +186 -0
  122. package/src/AgentBase.ts +2489 -0
  123. package/src/AgentServer.ts +415 -0
  124. package/src/AuthHandler.ts +256 -0
  125. package/src/ConfigLoader.ts +447 -0
  126. package/src/ContextBuilder.ts +1150 -0
  127. package/src/DataMap.ts +449 -0
  128. package/src/FunctionResult.ts +866 -0
  129. package/src/Logger.ts +337 -0
  130. package/src/PomBuilder.ts +439 -0
  131. package/src/PromptManager.ts +133 -0
  132. package/src/SWMLHandler.ts +298 -0
  133. package/src/SWMLService.ts +717 -0
  134. package/src/SchemaUtils.ts +361 -0
  135. package/src/SecurityUtils.ts +153 -0
  136. package/src/ServerlessAdapter.ts +221 -0
  137. package/src/SessionManager.ts +311 -0
  138. package/src/SslConfig.ts +118 -0
  139. package/src/SwaigFunction.ts +276 -0
  140. package/src/SwmlBuilder.ts +258 -0
  141. package/src/SwmlVerbMethods.generated.ts +234 -0
  142. package/src/TypeInference.ts +181 -0
  143. package/src/WebService.ts +619 -0
  144. package/src/cli/agent-loader.ts +166 -0
  145. package/src/cli/mock-data.ts +111 -0
  146. package/src/cli/swaig-test.ts +452 -0
  147. package/src/generateVerbTypes.ts +322 -0
  148. package/src/index.ts +167 -0
  149. package/src/livewire/index.ts +950 -0
  150. package/src/prefabs/ConciergeAgent.ts +287 -0
  151. package/src/prefabs/FAQBotAgent.ts +384 -0
  152. package/src/prefabs/InfoGathererAgent.ts +339 -0
  153. package/src/prefabs/ReceptionistAgent.ts +366 -0
  154. package/src/prefabs/SurveyAgent.ts +646 -0
  155. package/src/prefabs/index.ts +27 -0
  156. package/src/relay/Action.ts +314 -0
  157. package/src/relay/Call.ts +854 -0
  158. package/src/relay/Deferred.ts +71 -0
  159. package/src/relay/Message.ts +148 -0
  160. package/src/relay/RelayClient.ts +1036 -0
  161. package/src/relay/RelayError.ts +14 -0
  162. package/src/relay/RelayEvent.ts +892 -0
  163. package/src/relay/constants.ts +132 -0
  164. package/src/relay/index.ts +123 -0
  165. package/src/relay/types.ts +149 -0
  166. package/src/rest/HttpClient.ts +103 -0
  167. package/src/rest/RestError.ts +31 -0
  168. package/src/rest/base/BaseResource.ts +19 -0
  169. package/src/rest/base/CrudResource.ts +51 -0
  170. package/src/rest/base/CrudWithAddresses.ts +26 -0
  171. package/src/rest/index.ts +200 -0
  172. package/src/rest/namespaces/addresses.ts +32 -0
  173. package/src/rest/namespaces/calling.ts +187 -0
  174. package/src/rest/namespaces/chat.ts +20 -0
  175. package/src/rest/namespaces/compat.ts +352 -0
  176. package/src/rest/namespaces/datasphere.ts +45 -0
  177. package/src/rest/namespaces/fabric.ts +222 -0
  178. package/src/rest/namespaces/imported-numbers.ts +20 -0
  179. package/src/rest/namespaces/logs.ts +84 -0
  180. package/src/rest/namespaces/lookup.ts +21 -0
  181. package/src/rest/namespaces/mfa.ts +30 -0
  182. package/src/rest/namespaces/number-groups.ts +38 -0
  183. package/src/rest/namespaces/phone-numbers.ts +44 -0
  184. package/src/rest/namespaces/project.ts +36 -0
  185. package/src/rest/namespaces/pubsub.ts +20 -0
  186. package/src/rest/namespaces/queues.ts +33 -0
  187. package/src/rest/namespaces/recordings.ts +28 -0
  188. package/src/rest/namespaces/registry.ts +101 -0
  189. package/src/rest/namespaces/short-codes.ts +28 -0
  190. package/src/rest/namespaces/sip-profile.ts +25 -0
  191. package/src/rest/namespaces/verified-callers.ts +27 -0
  192. package/src/rest/namespaces/video.ts +165 -0
  193. package/src/rest/pagination.ts +84 -0
  194. package/src/rest/types.ts +61 -0
  195. package/src/schema.json +12250 -0
  196. package/src/skills/SkillBase.ts +547 -0
  197. package/src/skills/SkillManager.ts +415 -0
  198. package/src/skills/SkillRegistry.ts +367 -0
  199. package/src/skills/builtin/api_ninjas_trivia.ts +280 -0
  200. package/src/skills/builtin/ask_claude.ts +241 -0
  201. package/src/skills/builtin/claude_skills.ts +978 -0
  202. package/src/skills/builtin/custom_skills.ts +312 -0
  203. package/src/skills/builtin/datasphere.ts +438 -0
  204. package/src/skills/builtin/datasphere_serverless.ts +339 -0
  205. package/src/skills/builtin/datetime.ts +130 -0
  206. package/src/skills/builtin/google_maps.ts +651 -0
  207. package/src/skills/builtin/index.ts +88 -0
  208. package/src/skills/builtin/info_gatherer.ts +405 -0
  209. package/src/skills/builtin/joke.ts +191 -0
  210. package/src/skills/builtin/math.ts +142 -0
  211. package/src/skills/builtin/mcp_gateway.ts +677 -0
  212. package/src/skills/builtin/native_vector_search.ts +901 -0
  213. package/src/skills/builtin/play_background_file.ts +358 -0
  214. package/src/skills/builtin/spider.ts +869 -0
  215. package/src/skills/builtin/swml_transfer.ts +635 -0
  216. package/src/skills/builtin/weather_api.ts +248 -0
  217. package/src/skills/builtin/web_search.ts +1074 -0
  218. package/src/skills/builtin/wikipedia_search.ts +299 -0
  219. package/src/skills/index.ts +12 -0
  220. package/src/types.ts +132 -0
  221. package/tests/AgentBase.test.ts +1093 -0
  222. package/tests/AgentServer.test.ts +126 -0
  223. package/tests/AuthHandler.test.ts +133 -0
  224. package/tests/ConfigLoader.test.ts +151 -0
  225. package/tests/ContextBuilder.test.ts +108 -0
  226. package/tests/DataMap.test.ts +367 -0
  227. package/tests/EphemeralSkillCopy.test.ts +107 -0
  228. package/tests/FunctionResult.test.ts +298 -0
  229. package/tests/GracefulShutdown.test.ts +46 -0
  230. package/tests/Logger.test.ts +420 -0
  231. package/tests/PomBuilder.test.ts +228 -0
  232. package/tests/ProxyDetection.test.ts +228 -0
  233. package/tests/SWMLService.test.ts +320 -0
  234. package/tests/SWMLVerbHandler.test.ts +523 -0
  235. package/tests/SchemaUtils.test.ts +106 -0
  236. package/tests/SchemaUtils.verb.test.ts +206 -0
  237. package/tests/SecurityUtils.test.ts +192 -0
  238. package/tests/ServerlessAdapter.test.ts +222 -0
  239. package/tests/SessionManager.test.ts +257 -0
  240. package/tests/SkillBaseFeatures.test.ts +130 -0
  241. package/tests/SkillMultiInstance.test.ts +102 -0
  242. package/tests/SkillParameterSchemas.test.ts +134 -0
  243. package/tests/SkillRegistrySchema.test.ts +84 -0
  244. package/tests/SslConfig.test.ts +97 -0
  245. package/tests/SwmlBuilder.test.ts +400 -0
  246. package/tests/TypeInference.test.ts +165 -0
  247. package/tests/VerbHandlerRegistry.test.ts +128 -0
  248. package/tests/cli/mock-data.test.ts +113 -0
  249. package/tests/cli/swaig-test.test.ts +108 -0
  250. package/tests/examples.test.ts +194 -0
  251. package/tests/fixtures/scrape-parity/README.md +53 -0
  252. package/tests/fixtures/scrape-parity/expected/01_simple_article.json +5 -0
  253. package/tests/fixtures/scrape-parity/expected/02_class_based_noise.json +5 -0
  254. package/tests/fixtures/scrape-parity/expected/03_nested_containers.json +5 -0
  255. package/tests/fixtures/scrape-parity/expected/04_bare_body.json +5 -0
  256. package/tests/fixtures/scrape-parity/expected/05_malformed.json +5 -0
  257. package/tests/fixtures/scrape-parity/expected/06_multi_class.json +5 -0
  258. package/tests/fixtures/scrape-parity/expected/07_anchor_pattern.json +5 -0
  259. package/tests/fixtures/scrape-parity/expected/08_inline_script.json +5 -0
  260. package/tests/fixtures/scrape-parity/expected/09_whitespace_nbsp.json +5 -0
  261. package/tests/fixtures/scrape-parity/expected/10_entity_unicode.json +5 -0
  262. package/tests/fixtures/scrape-parity/expected/11_cdata_comments.json +5 -0
  263. package/tests/fixtures/scrape-parity/expected/12_deep_nested_removals.json +5 -0
  264. package/tests/fixtures/scrape-parity/html/01_simple_article.html +18 -0
  265. package/tests/fixtures/scrape-parity/html/02_class_based_noise.html +18 -0
  266. package/tests/fixtures/scrape-parity/html/03_nested_containers.html +13 -0
  267. package/tests/fixtures/scrape-parity/html/04_bare_body.html +9 -0
  268. package/tests/fixtures/scrape-parity/html/05_malformed.html +11 -0
  269. package/tests/fixtures/scrape-parity/html/06_multi_class.html +18 -0
  270. package/tests/fixtures/scrape-parity/html/07_anchor_pattern.html +12 -0
  271. package/tests/fixtures/scrape-parity/html/08_inline_script.html +17 -0
  272. package/tests/fixtures/scrape-parity/html/09_whitespace_nbsp.html +21 -0
  273. package/tests/fixtures/scrape-parity/html/10_entity_unicode.html +10 -0
  274. package/tests/fixtures/scrape-parity/html/11_cdata_comments.html +13 -0
  275. package/tests/fixtures/scrape-parity/html/12_deep_nested_removals.html +24 -0
  276. package/tests/fixtures/scrape-parity/python_extract.py +109 -0
  277. package/tests/livewire/livewire.test.ts +780 -0
  278. package/tests/mcp.test.ts +221 -0
  279. package/tests/prefabs/prefabs.test.ts +1093 -0
  280. package/tests/relay/Action.test.ts +392 -0
  281. package/tests/relay/Call.test.ts +535 -0
  282. package/tests/relay/Deferred.test.ts +71 -0
  283. package/tests/relay/Message.test.ts +164 -0
  284. package/tests/relay/RelayClient.test.ts +779 -0
  285. package/tests/relay/RelayEvent.test.ts +463 -0
  286. package/tests/relay/helpers.ts +79 -0
  287. package/tests/rest/HttpClient.test.ts +274 -0
  288. package/tests/rest/RestError.test.ts +53 -0
  289. package/tests/rest/calling.test.ts +156 -0
  290. package/tests/rest/client.test.ts +197 -0
  291. package/tests/rest/compat.test.ts +335 -0
  292. package/tests/rest/datasphere.test.ts +71 -0
  293. package/tests/rest/fabric.test.ts +186 -0
  294. package/tests/rest/helpers.ts +95 -0
  295. package/tests/rest/logs.test.ts +68 -0
  296. package/tests/rest/pagination.test.ts +133 -0
  297. package/tests/rest/registry.test.ts +94 -0
  298. package/tests/rest/relay-rest.test.ts +290 -0
  299. package/tests/rest/video.test.ts +173 -0
  300. package/tests/setup.ts +6 -0
  301. package/tests/skills/SkillBase.test.ts +108 -0
  302. package/tests/skills/SkillManager.test.ts +268 -0
  303. package/tests/skills/SkillRegistry.test.ts +176 -0
  304. package/tests/skills/builtin.test.ts +1472 -0
  305. package/tests/skills/claude-skills.test.ts +54 -0
  306. package/tests/skills/custom-skills.test.ts +53 -0
  307. package/tests/skills/datasphere-serverless.test.ts +70 -0
  308. package/tests/skills/datasphere.test.ts +107 -0
  309. package/tests/skills/datetime.test.ts +101 -0
  310. package/tests/skills/google-maps.test.ts +78 -0
  311. package/tests/skills/info-gatherer.test.ts +411 -0
  312. package/tests/skills/joke.test.ts +81 -0
  313. package/tests/skills/math.test.ts +68 -0
  314. package/tests/skills/mcp-gateway.test.ts +101 -0
  315. package/tests/skills/play-background.test.ts +54 -0
  316. package/tests/skills/spider.test.ts +108 -0
  317. package/tests/skills/swml-transfer.test.ts +178 -0
  318. package/tests/skills/trivia.test.ts +54 -0
  319. package/tests/skills/vector-search.test.ts +135 -0
  320. package/tests/skills/weather-api.test.ts +77 -0
  321. package/tests/skills/web-search-parity.test.ts +59 -0
  322. package/tests/skills/web-search.test.ts +103 -0
  323. package/tests/skills/wikipedia.test.ts +63 -0
  324. package/tsconfig.json +18 -0
  325. package/vitest.config.ts +9 -0
@@ -0,0 +1,869 @@
1
+ /**
2
+ * Spider Skill - Fast web scraping and crawling capabilities.
3
+ *
4
+ * Tier 2 built-in skill. Port of the Python `SpiderSkill` that performs
5
+ * HTTP scraping via `fetch`, lightweight HTML-to-text extraction, and
6
+ * breadth-first crawling with configurable limits. Supports three tools:
7
+ *
8
+ * - `scrape_url` — single-page text/markdown extraction
9
+ * - `crawl_site` — breadth-first crawl from a start URL
10
+ * - `extract_structured_data` — CSS/XPath-like structured extraction
11
+ *
12
+ * Multiple instances are supported; use `tool_name` in config to differentiate.
13
+ */
14
+
15
+ import { SkillBase } from '../SkillBase.js';
16
+ import type {
17
+ SkillToolDefinition,
18
+ SkillConfig,
19
+ ParameterSchemaEntry,
20
+ } from '../SkillBase.js';
21
+ import { FunctionResult } from '../../FunctionResult.js';
22
+ import { resolveAndValidateUrl, validateUrl, MAX_SKILL_INPUT_LENGTH } from '../../SecurityUtils.js';
23
+ import { getLogger } from '../../Logger.js';
24
+ import * as cheerio from 'cheerio';
25
+
26
+ const log = getLogger('SpiderSkill');
27
+
28
+ /** Cached response entry used by the internal LRU cache. */
29
+ interface CachedResponse {
30
+ /** Resolved URL (after redirects). */
31
+ url: string;
32
+ /** HTTP status code. */
33
+ status: number;
34
+ /** Raw HTML body (or best-effort text for non-HTML). */
35
+ body: string;
36
+ }
37
+
38
+ const WHITESPACE_REGEX = /\s+/g;
39
+
40
+ /** Tags stripped from HTML before text extraction. */
41
+ const STRIP_TAG_REGEXES: RegExp[] = [
42
+ /<script\b[^>]*>[\s\S]*?<\/script>/gi,
43
+ /<style\b[^>]*>[\s\S]*?<\/style>/gi,
44
+ /<nav\b[^>]*>[\s\S]*?<\/nav>/gi,
45
+ /<header\b[^>]*>[\s\S]*?<\/header>/gi,
46
+ /<footer\b[^>]*>[\s\S]*?<\/footer>/gi,
47
+ /<aside\b[^>]*>[\s\S]*?<\/aside>/gi,
48
+ /<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi,
49
+ ];
50
+
51
+ /**
52
+ * Fast web scraping skill optimized for speed and token efficiency.
53
+ *
54
+ * Multi-instance capable. Port of the Python `SpiderSkill` with three tools:
55
+ * `scrape_url`, `crawl_site`, and `extract_structured_data`. Configuration
56
+ * mirrors the Python schema (delay, concurrent_requests, timeout, max_pages,
57
+ * max_depth, extract_type, max_text_length, clean_text, selectors,
58
+ * follow_patterns, user_agent, headers, follow_robots_txt, cache_enabled).
59
+ */
60
+ export class SpiderSkill extends SkillBase {
61
+ // Python ground truth: skills/spider/skill.py:~140-145
62
+ // REQUIRED_PACKAGES = ["lxml"] in Python; TS uses cheerio.
63
+ static override SKILL_NAME = 'spider';
64
+ static override SKILL_DESCRIPTION = 'Fast web scraping and crawling capabilities';
65
+ static override SKILL_VERSION = '1.0.0';
66
+ static override REQUIRED_PACKAGES: readonly string[] = ['cheerio'];
67
+ static override REQUIRED_ENV_VARS: readonly string[] = [];
68
+ static override SUPPORTS_MULTIPLE_INSTANCES = true;
69
+
70
+ static override getParameterSchema(): Record<string, ParameterSchemaEntry> {
71
+ return {
72
+ ...super.getParameterSchema(),
73
+ delay: {
74
+ type: 'number',
75
+ description: 'Delay between requests in seconds',
76
+ default: 0.1,
77
+ required: false,
78
+ min: 0,
79
+ },
80
+ concurrent_requests: {
81
+ type: 'integer',
82
+ description: 'Number of concurrent requests allowed',
83
+ default: 5,
84
+ required: false,
85
+ min: 1,
86
+ max: 20,
87
+ },
88
+ timeout: {
89
+ type: 'integer',
90
+ description: 'Request timeout in seconds',
91
+ default: 5,
92
+ required: false,
93
+ min: 1,
94
+ max: 60,
95
+ },
96
+ max_pages: {
97
+ type: 'integer',
98
+ description: 'Maximum number of pages to scrape',
99
+ default: 1,
100
+ required: false,
101
+ min: 1,
102
+ max: 100,
103
+ },
104
+ max_depth: {
105
+ type: 'integer',
106
+ description: 'Maximum crawl depth (0 = single page only)',
107
+ default: 0,
108
+ required: false,
109
+ min: 0,
110
+ max: 5,
111
+ },
112
+ extract_type: {
113
+ type: 'string',
114
+ description: 'Content extraction method',
115
+ default: 'fast_text',
116
+ required: false,
117
+ enum: ['fast_text', 'clean_text', 'full_text', 'html', 'markdown', 'structured', 'custom'],
118
+ },
119
+ max_text_length: {
120
+ type: 'integer',
121
+ description: 'Maximum text length to return',
122
+ default: 3000,
123
+ required: false,
124
+ min: 100,
125
+ max: 100000,
126
+ },
127
+ clean_text: {
128
+ type: 'boolean',
129
+ description: 'Whether to clean extracted text',
130
+ default: true,
131
+ required: false,
132
+ },
133
+ selectors: {
134
+ type: 'object',
135
+ description: 'Custom CSS/XPath selectors for structured extraction',
136
+ default: {},
137
+ required: false,
138
+ additionalProperties: { type: 'string' },
139
+ } as ParameterSchemaEntry & { additionalProperties?: unknown },
140
+ follow_patterns: {
141
+ type: 'array',
142
+ description: 'URL patterns to follow when crawling',
143
+ default: [],
144
+ required: false,
145
+ items: { type: 'string' },
146
+ },
147
+ user_agent: {
148
+ type: 'string',
149
+ description: 'User agent string for requests',
150
+ default: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
151
+ required: false,
152
+ },
153
+ headers: {
154
+ type: 'object',
155
+ description: 'Additional HTTP headers',
156
+ default: {},
157
+ required: false,
158
+ additionalProperties: { type: 'string' },
159
+ } as ParameterSchemaEntry & { additionalProperties?: unknown },
160
+ follow_robots_txt: {
161
+ type: 'boolean',
162
+ // Matches Python's __init__ runtime fallback (skills/spider/skill.py:177
163
+ // uses `params.get('follow_robots_txt', False)`). Python's schema said
164
+ // True but the runtime and TS both use False — aligning the schema
165
+ // eliminates the cross-SDK contract mismatch.
166
+ description: 'Whether to respect robots.txt (default: false to match Python runtime)',
167
+ default: false,
168
+ required: false,
169
+ },
170
+ cache_enabled: {
171
+ type: 'boolean',
172
+ description: 'Whether to cache scraped pages',
173
+ default: true,
174
+ required: false,
175
+ },
176
+ };
177
+ }
178
+
179
+ // Runtime state, populated in setup()
180
+ private delay = 0.1;
181
+ private concurrentRequests = 5;
182
+ private timeout = 5;
183
+ private maxPages = 1;
184
+ private maxDepth = 0;
185
+ private extractType = 'fast_text';
186
+ private maxTextLength = 3000;
187
+ private cleanText = true;
188
+ private cacheEnabled = true;
189
+ private followRobotsTxt = true;
190
+ private userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
191
+ private headers: Record<string, string> = {};
192
+ private selectors: Record<string, string> = {};
193
+ private compiledFollowPatterns: RegExp[] = [];
194
+ private cache: Map<string, CachedResponse> | null = null;
195
+ private readonly cacheMaxSize = 100;
196
+
197
+ override getInstanceKey(): string {
198
+ const toolName = this.getConfig<string>('tool_name', this.skillName);
199
+ return `${this.skillName}_${toolName}`;
200
+ }
201
+
202
+ override async setup(): Promise<boolean> {
203
+ // Performance
204
+ this.delay = this.getConfig<number>('delay', 0.1);
205
+ this.concurrentRequests = this.getConfig<number>('concurrent_requests', 5);
206
+ this.timeout = this.getConfig<number>('timeout', 5);
207
+
208
+ // Crawl limits
209
+ this.maxPages = this.getConfig<number>('max_pages', 1);
210
+ this.maxDepth = this.getConfig<number>('max_depth', 0);
211
+
212
+ // Content processing
213
+ this.extractType = this.getConfig<string>('extract_type', 'fast_text');
214
+ // Python has an internal inconsistency (schema: 10000, __init__ fallback:
215
+ // 3000). The effective runtime default is 3000 because Python reads via
216
+ // `self.params.get('max_text_length', 3000)` and schema defaults are not
217
+ // applied. Use 3000 for parity. Follow-up: reconcile Python schema vs init.
218
+ this.maxTextLength = this.getConfig<number>('max_text_length', 3000);
219
+ this.cleanText = this.getConfig<boolean>('clean_text', true);
220
+
221
+ // Features
222
+ this.cacheEnabled = this.getConfig<boolean>('cache_enabled', true);
223
+ this.followRobotsTxt = this.getConfig<boolean>('follow_robots_txt', false);
224
+ this.userAgent = this.getConfig<string>(
225
+ 'user_agent',
226
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
227
+ );
228
+
229
+ // Optional headers + user-agent merge
230
+ const headers = this.getConfig<Record<string, string>>('headers', {}) ?? {};
231
+ this.headers = { ...headers, 'User-Agent': this.userAgent };
232
+
233
+ // Selectors for structured extraction
234
+ this.selectors = this.getConfig<Record<string, string>>('selectors', {}) ?? {};
235
+
236
+ // Setup cache
237
+ this.cache = this.cacheEnabled ? new Map() : null;
238
+
239
+ // Validate numeric ranges
240
+ if (this.delay < 0) {
241
+ log.error('spider: delay cannot be negative', { delay: this.delay });
242
+ return false;
243
+ }
244
+ if (this.concurrentRequests < 1 || this.concurrentRequests > 20) {
245
+ log.error(
246
+ 'spider: concurrent_requests must be between 1 and 20',
247
+ { concurrent_requests: this.concurrentRequests },
248
+ );
249
+ return false;
250
+ }
251
+ if (this.maxPages < 1) {
252
+ log.error('spider: max_pages must be at least 1', { max_pages: this.maxPages });
253
+ return false;
254
+ }
255
+ if (this.maxDepth < 0) {
256
+ log.error('spider: max_depth cannot be negative', { max_depth: this.maxDepth });
257
+ return false;
258
+ }
259
+
260
+ // Pre-compile follow patterns
261
+ this.compiledFollowPatterns = [];
262
+ const patterns = this.getConfig<string[]>('follow_patterns', []) ?? [];
263
+ for (const pattern of patterns) {
264
+ try {
265
+ this.compiledFollowPatterns.push(new RegExp(pattern));
266
+ } catch (err) {
267
+ log.error('spider: invalid follow pattern', {
268
+ pattern,
269
+ error: err instanceof Error ? err.message : String(err),
270
+ });
271
+ }
272
+ }
273
+
274
+ log.info('spider: configured', {
275
+ delay: this.delay,
276
+ max_pages: this.maxPages,
277
+ max_depth: this.maxDepth,
278
+ });
279
+ return true;
280
+ }
281
+
282
+ override getHints(): string[] {
283
+ return [
284
+ 'scrape',
285
+ 'crawl',
286
+ 'extract',
287
+ 'web page',
288
+ 'website',
289
+ 'get content from',
290
+ 'fetch data from',
291
+ 'spider',
292
+ ];
293
+ }
294
+
295
+ override async cleanup(): Promise<void> {
296
+ if (this.cache) {
297
+ this.cache.clear();
298
+ }
299
+ log.info('spider: cleaned up');
300
+ }
301
+
302
+ /** @returns Three tools: `scrape_url`, `crawl_site`, and `extract_structured_data`. */
303
+ getTools(): SkillToolDefinition[] {
304
+ const toolPrefix = this.getConfig<string>('tool_name', '');
305
+ const prefix = toolPrefix ? `${toolPrefix}_` : '';
306
+
307
+ return [
308
+ {
309
+ name: `${prefix}scrape_url`,
310
+ description: 'Extract text content from a single web page',
311
+ parameters: {
312
+ url: {
313
+ type: 'string',
314
+ description: 'The URL to scrape',
315
+ },
316
+ },
317
+ required: ['url'],
318
+ handler: async (
319
+ args: Record<string, unknown>,
320
+ rawData: Record<string, unknown>,
321
+ ) => this._scrapeUrlHandler(args, rawData),
322
+ },
323
+ {
324
+ name: `${prefix}crawl_site`,
325
+ description: 'Crawl multiple pages starting from a URL',
326
+ parameters: {
327
+ start_url: {
328
+ type: 'string',
329
+ description: 'Starting URL for the crawl',
330
+ },
331
+ },
332
+ required: ['start_url'],
333
+ handler: async (
334
+ args: Record<string, unknown>,
335
+ rawData: Record<string, unknown>,
336
+ ) => this._crawlSiteHandler(args, rawData),
337
+ },
338
+ {
339
+ name: `${prefix}extract_structured_data`,
340
+ description: 'Extract specific data from a web page using selectors',
341
+ parameters: {
342
+ url: {
343
+ type: 'string',
344
+ description: 'The URL to scrape',
345
+ },
346
+ },
347
+ required: ['url'],
348
+ handler: async (
349
+ args: Record<string, unknown>,
350
+ rawData: Record<string, unknown>,
351
+ ) => this._extractStructuredHandler(args, rawData),
352
+ },
353
+ ];
354
+ }
355
+
356
+ // ---------------------------------------------------------------------------
357
+ // Internal helpers
358
+ // ---------------------------------------------------------------------------
359
+
360
+ /** Fetch a URL with caching and timeout handling. Returns null on failure. */
361
+ private async _fetchUrl(url: string): Promise<CachedResponse | null> {
362
+ if (this.cacheEnabled && this.cache?.has(url)) {
363
+ log.debug('spider: cache hit', { url });
364
+ return this.cache.get(url)!;
365
+ }
366
+
367
+ const controller = new AbortController();
368
+ const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
369
+ try {
370
+ const response = await fetch(url, {
371
+ method: 'GET',
372
+ headers: this.headers,
373
+ signal: controller.signal,
374
+ });
375
+
376
+ if (!response.ok) {
377
+ log.error('spider: fetch failed', { url, status: response.status });
378
+ return null;
379
+ }
380
+
381
+ const body = await response.text();
382
+ const cached: CachedResponse = {
383
+ url: response.url || url,
384
+ status: response.status,
385
+ body,
386
+ };
387
+
388
+ if (this.cacheEnabled && this.cache) {
389
+ if (this.cache.size >= this.cacheMaxSize) {
390
+ const oldest = this.cache.keys().next().value;
391
+ if (oldest !== undefined) this.cache.delete(oldest);
392
+ }
393
+ this.cache.set(url, cached);
394
+ }
395
+ return cached;
396
+ } catch (err) {
397
+ log.error('spider: fetch error', {
398
+ url,
399
+ error: err instanceof Error ? err.message : String(err),
400
+ });
401
+ return null;
402
+ } finally {
403
+ clearTimeout(timer);
404
+ }
405
+ }
406
+
407
+ /**
408
+ * Extract plain text from a fetched response using simple regex stripping.
409
+ *
410
+ * Takes a {@link CachedResponse} (TS equivalent of `requests.Response`) to
411
+ * match Python's `_fast_text_extract(response)` capability surface — the
412
+ * response-like object exposes `url`, `status`, and `body` for callers that
413
+ * need to branch on status/content-type.
414
+ */
415
+ private _fastTextExtract(response: CachedResponse): string {
416
+ try {
417
+ // Parse through cheerio so every named and numeric HTML entity is
418
+ // decoded (Python's lxml does this natively). Previously TS only
419
+ // decoded six hand-coded entities, leaving `&mdash;`, `&#8212;`, etc.
420
+ // literal in the output.
421
+ const $ = cheerio.load(response.body);
422
+ // Strip noise tags before extracting text — matches Python's
423
+ // lxml drop_tree() on the same 7 tags.
424
+ for (const tag of [
425
+ 'script',
426
+ 'style',
427
+ 'nav',
428
+ 'header',
429
+ 'footer',
430
+ 'aside',
431
+ 'noscript',
432
+ ]) {
433
+ $(tag).remove();
434
+ }
435
+ let text = $('body').text();
436
+ if (!text || text.trim().length === 0) {
437
+ // Pages without <body> fall through to document-root text.
438
+ text = $.root().text();
439
+ }
440
+ if (this.cleanText) {
441
+ text = text.replace(WHITESPACE_REGEX, ' ').trim();
442
+ }
443
+
444
+ // Smart truncation
445
+ if (text.length > this.maxTextLength) {
446
+ const keepStart = Math.floor((this.maxTextLength * 2) / 3);
447
+ const keepEnd = Math.floor(this.maxTextLength / 3);
448
+ text =
449
+ text.slice(0, keepStart) +
450
+ '\n\n[...CONTENT TRUNCATED...]\n\n' +
451
+ text.slice(-keepEnd);
452
+ }
453
+
454
+ return text;
455
+ } catch (err) {
456
+ // Python swallows extraction errors and returns '' so the handler
457
+ // emits a clean "no content extracted" message instead of bubbling
458
+ // a parse failure all the way up to the AI.
459
+ log.error('spider: fast-text extraction failed', {
460
+ error: err instanceof Error ? err.message : String(err),
461
+ });
462
+ return '';
463
+ }
464
+ }
465
+
466
+ /**
467
+ * Extract a value using a CSS selector (via cheerio) or XPath-like
468
+ * selector (`//tag`). Mirrors Python `_structured_extract`'s selector
469
+ * branching: selectors starting with `/` are treated as XPath (tag-name
470
+ * subset), everything else is full CSS via cheerio's querySelector-style
471
+ * engine.
472
+ */
473
+ private _applySelector(
474
+ $: cheerio.CheerioAPI,
475
+ selector: string,
476
+ ): string[] {
477
+ const trimmed = selector.trim();
478
+ if (!trimmed) return [];
479
+
480
+ if (trimmed.startsWith('/')) {
481
+ // Subset of XPath supported in Python's lxml path — `//tag`, `//tag/text()`.
482
+ const tagMatch = /\/\/([a-zA-Z][a-zA-Z0-9]*)/.exec(trimmed);
483
+ if (!tagMatch) return [];
484
+ return $(tagMatch[1])
485
+ .toArray()
486
+ .map((el) => $(el).text().trim())
487
+ .filter((t) => t.length > 0);
488
+ }
489
+
490
+ return $(trimmed)
491
+ .toArray()
492
+ .map((el) => $(el).text().trim())
493
+ .filter((t) => t.length > 0);
494
+ }
495
+
496
+ /** Markdown extraction using cheerio. Mirrors Python `_markdown_extract`. */
497
+ private _markdownExtract(response: CachedResponse): string {
498
+ try {
499
+ const $ = cheerio.load(response.body);
500
+
501
+ // Remove unwanted tags
502
+ for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside']) {
503
+ $(tag).remove();
504
+ }
505
+
506
+ const parts: string[] = [];
507
+ const title = $('title').first().text().trim();
508
+ if (title) parts.push(`# ${title}\n`);
509
+
510
+ $('h1, h2, h3, h4, h5, h6, p, li, code, pre').each((_, el) => {
511
+ const tag = (el as { tagName: string }).tagName.toLowerCase();
512
+ const text = $(el).text().trim();
513
+ if (!text) return;
514
+ if (/^h[1-6]$/.test(tag)) {
515
+ const level = Number(tag[1]);
516
+ parts.push(`\n${'#'.repeat(level)} ${text}\n`);
517
+ } else if (tag === 'p') {
518
+ parts.push(`\n${text}\n`);
519
+ } else if (tag === 'li') {
520
+ parts.push(`- ${text}`);
521
+ } else if (tag === 'code' || tag === 'pre') {
522
+ parts.push(`\n\`\`\`\n${text}\n\`\`\`\n`);
523
+ }
524
+ });
525
+
526
+ let text = parts.join('\n');
527
+ if (text.length > this.maxTextLength) {
528
+ text = text.slice(0, this.maxTextLength) + '\n\n[...TRUNCATED...]';
529
+ }
530
+ return text;
531
+ } catch (err) {
532
+ log.error('spider: markdown extraction error', {
533
+ error: err instanceof Error ? err.message : String(err),
534
+ });
535
+ return this._fastTextExtract(response);
536
+ }
537
+ }
538
+
539
+ private _structuredExtract(
540
+ cached: CachedResponse,
541
+ selectors: Record<string, string>,
542
+ ): Record<string, unknown> {
543
+ let $: cheerio.CheerioAPI;
544
+ try {
545
+ $ = cheerio.load(cached.body);
546
+ } catch (err) {
547
+ // Python returns `{'error': str(e)}` from _structured_extract when the
548
+ // parser fails (skill.py). Surface the same shape so upstream handlers
549
+ // can branch on `'error' in result`.
550
+ log.error('spider: structured parse failed', {
551
+ error: err instanceof Error ? err.message : String(err),
552
+ });
553
+ return { error: err instanceof Error ? err.message : String(err) };
554
+ }
555
+ const result: Record<string, unknown> = {
556
+ url: cached.url,
557
+ status_code: cached.status,
558
+ title: $('title').first().text().trim(),
559
+ data: {},
560
+ };
561
+
562
+ const data: Record<string, unknown> = {};
563
+ for (const [field, selector] of Object.entries(selectors)) {
564
+ try {
565
+ const values = this._applySelector($, selector);
566
+ if (values.length === 0) {
567
+ data[field] = null;
568
+ } else if (values.length === 1) {
569
+ data[field] = values[0];
570
+ } else {
571
+ data[field] = values;
572
+ }
573
+ } catch (err) {
574
+ log.warn('spider: selector error', {
575
+ selector,
576
+ error: err instanceof Error ? err.message : String(err),
577
+ });
578
+ data[field] = null;
579
+ }
580
+ }
581
+ result['data'] = data;
582
+ return result;
583
+ }
584
+
585
+ // ---------------------------------------------------------------------------
586
+ // Tool handlers
587
+ // ---------------------------------------------------------------------------
588
+
589
+ private async _scrapeUrlHandler(
590
+ args: Record<string, unknown>,
591
+ _rawData: Record<string, unknown>,
592
+ ): Promise<FunctionResult> {
593
+ const urlArg = args['url'];
594
+ if (typeof urlArg !== 'string' || urlArg.trim().length === 0) {
595
+ return new FunctionResult('Please provide a URL to scrape');
596
+ }
597
+ const url = urlArg.trim();
598
+
599
+ if (url.length > MAX_SKILL_INPUT_LENGTH) {
600
+ return new FunctionResult('Input URL is too long.');
601
+ }
602
+
603
+ if (!/^https?:\/\//.test(url)) {
604
+ return new FunctionResult(`Invalid URL: ${url}`);
605
+ }
606
+
607
+ // SSRF protection
608
+ const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
609
+ try {
610
+ await resolveAndValidateUrl(url, allowPrivate);
611
+ } catch {
612
+ return new FunctionResult(
613
+ 'URL rejected: cannot access private or internal URLs',
614
+ );
615
+ }
616
+
617
+ const cached = await this._fetchUrl(url);
618
+ if (!cached) {
619
+ return new FunctionResult(`Failed to fetch ${url}`);
620
+ }
621
+
622
+ try {
623
+ if (this.extractType === 'structured') {
624
+ const result = this._structuredExtract(cached, this.selectors);
625
+ return new FunctionResult(
626
+ `Extracted structured data from ${url}: ${JSON.stringify(result)}`,
627
+ );
628
+ }
629
+ const content =
630
+ this.extractType === 'markdown'
631
+ ? this._markdownExtract(cached)
632
+ : this._fastTextExtract(cached);
633
+
634
+ if (!content) {
635
+ return new FunctionResult(`No content extracted from ${url}`);
636
+ }
637
+
638
+ const charCount = content.length;
639
+ const header = `Content from ${url} (${charCount} characters):\n\n`;
640
+ return new FunctionResult(header + content);
641
+ } catch (err) {
642
+ log.error('spider: scrape_url error', {
643
+ url,
644
+ error: err instanceof Error ? err.message : String(err),
645
+ });
646
+ return new FunctionResult(
647
+ `Error processing ${url}: ${err instanceof Error ? err.message : String(err)}`,
648
+ );
649
+ }
650
+ }
651
+
652
+ private async _crawlSiteHandler(
653
+ args: Record<string, unknown>,
654
+ _rawData: Record<string, unknown>,
655
+ ): Promise<FunctionResult> {
656
+ const startArg = args['start_url'];
657
+ if (typeof startArg !== 'string' || startArg.trim().length === 0) {
658
+ return new FunctionResult('Please provide a starting URL for the crawl');
659
+ }
660
+ const startUrl = startArg.trim();
661
+
662
+ if (!/^https?:\/\//.test(startUrl)) {
663
+ return new FunctionResult(`Invalid URL: ${startUrl}`);
664
+ }
665
+
666
+ const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
667
+ try {
668
+ await resolveAndValidateUrl(startUrl, allowPrivate);
669
+ } catch {
670
+ return new FunctionResult(
671
+ 'URL rejected: cannot access private or internal URLs',
672
+ );
673
+ }
674
+
675
+ const maxDepth = this.maxDepth;
676
+ const maxPages = this.maxPages;
677
+
678
+ if (maxDepth < 0) return new FunctionResult('Max depth cannot be negative');
679
+ if (maxPages < 1) return new FunctionResult('Max pages must be at least 1');
680
+
681
+ const visited = new Set<string>();
682
+ const toVisit: [string, number][] = [[startUrl, 0]];
683
+ const results: {
684
+ url: string;
685
+ depth: number;
686
+ contentLength: number;
687
+ summary: string;
688
+ }[] = [];
689
+
690
+ let startHost: string;
691
+ try {
692
+ startHost = new URL(startUrl).host;
693
+ } catch {
694
+ return new FunctionResult(`Invalid URL: ${startUrl}`);
695
+ }
696
+
697
+ while (toVisit.length > 0 && visited.size < maxPages) {
698
+ const next = toVisit.shift();
699
+ if (!next) break;
700
+ const [url, depth] = next;
701
+
702
+ if (visited.has(url) || depth > maxDepth) continue;
703
+
704
+ const cached = await this._fetchUrl(url);
705
+ if (!cached) continue;
706
+
707
+ visited.add(url);
708
+
709
+ const content = this._fastTextExtract(cached);
710
+ if (content) {
711
+ results.push({
712
+ url,
713
+ depth,
714
+ contentLength: content.length,
715
+ summary:
716
+ content.length > 500 ? content.slice(0, 500) + '...' : content,
717
+ });
718
+ }
719
+
720
+ // Extract links if not at max depth. Use cheerio so we correctly
721
+ // pick up unquoted and oddly-whitespaced href attributes that the
722
+ // regex path missed (Python skills/spider/skill.py:487 uses lxml
723
+ // xpath('//a[@href]/@href') which has the same robustness).
724
+ if (depth < maxDepth) {
725
+ try {
726
+ const $links = cheerio.load(cached.body);
727
+ const hrefs: string[] = [];
728
+ $links('a[href]').each((_, el) => {
729
+ const href = $links(el).attr('href');
730
+ if (href) hrefs.push(href);
731
+ });
732
+ for (const href of hrefs) {
733
+ let absoluteUrl: string;
734
+ try {
735
+ absoluteUrl = new URL(href, url).toString();
736
+ } catch {
737
+ continue;
738
+ }
739
+
740
+ if (this.compiledFollowPatterns.length > 0) {
741
+ const allowed = this.compiledFollowPatterns.some((re) =>
742
+ re.test(absoluteUrl),
743
+ );
744
+ if (!allowed) continue;
745
+ }
746
+
747
+ try {
748
+ const absHost = new URL(absoluteUrl).host;
749
+ if (absHost !== startHost) continue;
750
+ } catch {
751
+ continue;
752
+ }
753
+
754
+ // SSRF hardening — discovered links must pass the same
755
+ // validation the entry URL did. Prevents a crafted page from
756
+ // bouncing the crawler into internal/metadata endpoints.
757
+ if (!(await validateUrl(absoluteUrl))) {
758
+ log.debug('spider: skipping discovered URL rejected by SSRF', {
759
+ url: absoluteUrl,
760
+ });
761
+ continue;
762
+ }
763
+
764
+ if (!visited.has(absoluteUrl)) {
765
+ toVisit.push([absoluteUrl, depth + 1]);
766
+ }
767
+ }
768
+ } catch (err) {
769
+ log.warn('spider: link extraction error', {
770
+ url,
771
+ error: err instanceof Error ? err.message : String(err),
772
+ });
773
+ }
774
+ }
775
+
776
+ if (this.delay > 0 && visited.size < maxPages) {
777
+ await new Promise((resolve) => setTimeout(resolve, this.delay * 1000));
778
+ }
779
+ }
780
+
781
+ if (results.length === 0) {
782
+ return new FunctionResult(`No pages could be crawled from ${startUrl}`);
783
+ }
784
+
785
+ const lines: string[] = [
786
+ `Crawled ${results.length} pages from ${startHost}:`,
787
+ '',
788
+ ];
789
+ results.forEach((r, i) => {
790
+ lines.push(
791
+ `${i + 1}. ${r.url} (depth: ${r.depth}, ${r.contentLength} chars)`,
792
+ );
793
+ lines.push(` Summary: ${r.summary.slice(0, 100)}...`);
794
+ lines.push('');
795
+ });
796
+ const totalChars = results.reduce((acc, r) => acc + r.contentLength, 0);
797
+ lines.push('');
798
+ lines.push(
799
+ `Total content: ${totalChars.toLocaleString()} characters across ${results.length} pages`,
800
+ );
801
+
802
+ return new FunctionResult(lines.join('\n'));
803
+ }
804
+
805
+ private async _extractStructuredHandler(
806
+ args: Record<string, unknown>,
807
+ _rawData: Record<string, unknown>,
808
+ ): Promise<FunctionResult> {
809
+ const urlArg = args['url'];
810
+ if (typeof urlArg !== 'string' || urlArg.trim().length === 0) {
811
+ return new FunctionResult('Please provide a URL');
812
+ }
813
+ const url = urlArg.trim();
814
+
815
+ if (!/^https?:\/\//.test(url)) {
816
+ return new FunctionResult(`Invalid URL: ${url}`);
817
+ }
818
+
819
+ const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
820
+ try {
821
+ await resolveAndValidateUrl(url, allowPrivate);
822
+ } catch {
823
+ return new FunctionResult(
824
+ 'URL rejected: cannot access private or internal URLs',
825
+ );
826
+ }
827
+
828
+ if (!this.selectors || Object.keys(this.selectors).length === 0) {
829
+ return new FunctionResult(
830
+ 'No selectors configured for structured data extraction',
831
+ );
832
+ }
833
+
834
+ const cached = await this._fetchUrl(url);
835
+ if (!cached) {
836
+ return new FunctionResult(`Failed to fetch ${url}`);
837
+ }
838
+
839
+ const result = this._structuredExtract(cached, this.selectors);
840
+ if ('error' in result) {
841
+ return new FunctionResult(`Error extracting data: ${result['error']}`);
842
+ }
843
+
844
+ const lines: string[] = [`Extracted data from ${url}:`, ''];
845
+ lines.push(`Title: ${(result['title'] as string) || 'N/A'}`);
846
+ lines.push('');
847
+
848
+ const data = result['data'] as Record<string, unknown>;
849
+ if (data && Object.keys(data).length > 0) {
850
+ lines.push('Data:');
851
+ for (const [field, value] of Object.entries(data)) {
852
+ lines.push(`- ${field}: ${String(value)}`);
853
+ }
854
+ } else {
855
+ lines.push('No data extracted with provided selectors');
856
+ }
857
+
858
+ return new FunctionResult(lines.join('\n'));
859
+ }
860
+ }
861
+
862
+ /**
863
+ * Factory function for creating SpiderSkill instances.
864
+ * @param config - Optional skill configuration.
865
+ * @returns A new SpiderSkill instance.
866
+ */
867
+ export function createSkill(config?: SkillConfig): SpiderSkill {
868
+ return new SpiderSkill(config);
869
+ }