@signalwire/sdk 2.0.4 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +785 -0
  2. package/LICENSE +21 -0
  3. package/README.md +163 -68
  4. package/dist/AGENTS.md +13 -0
  5. package/dist/AgentBase.d.ts +550 -163
  6. package/dist/AgentBase.js +1640 -514
  7. package/dist/AgentBase.js.map +1 -1
  8. package/dist/AgentServer.d.ts +21 -0
  9. package/dist/AgentServer.js +154 -47
  10. package/dist/AgentServer.js.map +1 -1
  11. package/dist/AuthHandler.d.ts +14 -2
  12. package/dist/AuthHandler.js +7 -2
  13. package/dist/AuthHandler.js.map +1 -1
  14. package/dist/ConfigLoader.d.ts +5 -13
  15. package/dist/ConfigLoader.js +15 -29
  16. package/dist/ConfigLoader.js.map +1 -1
  17. package/dist/ContextBuilder.d.ts +202 -9
  18. package/dist/ContextBuilder.js +219 -36
  19. package/dist/ContextBuilder.js.map +1 -1
  20. package/dist/DataMap.d.ts +158 -45
  21. package/dist/DataMap.js +167 -52
  22. package/dist/DataMap.js.map +1 -1
  23. package/dist/FunctionResult.d.ts +277 -83
  24. package/dist/FunctionResult.js +444 -116
  25. package/dist/FunctionResult.js.map +1 -1
  26. package/dist/Logger.d.ts +30 -5
  27. package/dist/Logger.js +77 -19
  28. package/dist/Logger.js.map +1 -1
  29. package/dist/POM/PromptObjectModel.d.ts +163 -0
  30. package/dist/POM/PromptObjectModel.js +481 -0
  31. package/dist/POM/PromptObjectModel.js.map +1 -0
  32. package/dist/ParameterSchema.d.ts +306 -0
  33. package/dist/ParameterSchema.js +251 -0
  34. package/dist/ParameterSchema.js.map +1 -0
  35. package/dist/PlatformContracts.d.ts +21 -0
  36. package/dist/PlatformContracts.generated.d.ts +109 -0
  37. package/dist/PlatformContracts.generated.js +8 -0
  38. package/dist/PlatformContracts.generated.js.map +1 -0
  39. package/dist/PlatformContracts.js +2 -0
  40. package/dist/PlatformContracts.js.map +1 -0
  41. package/dist/PomBuilder.d.ts +28 -0
  42. package/dist/PomBuilder.js +81 -3
  43. package/dist/PomBuilder.js.map +1 -1
  44. package/dist/PostPrompt.d.ts +110 -0
  45. package/dist/PostPrompt.js +204 -0
  46. package/dist/PostPrompt.js.map +1 -0
  47. package/dist/PromptManager.d.ts +29 -1
  48. package/dist/PromptManager.js +37 -1
  49. package/dist/PromptManager.js.map +1 -1
  50. package/dist/PublicFetch.d.ts +132 -0
  51. package/dist/PublicFetch.js +311 -0
  52. package/dist/PublicFetch.js.map +1 -0
  53. package/dist/RobotsTxt.d.ts +41 -0
  54. package/dist/RobotsTxt.js +152 -0
  55. package/dist/RobotsTxt.js.map +1 -0
  56. package/dist/SWMLHandler.js.map +1 -1
  57. package/dist/SWMLService.d.ts +265 -66
  58. package/dist/SWMLService.js +652 -112
  59. package/dist/SWMLService.js.map +1 -1
  60. package/dist/SchemaUtils.d.ts +115 -2
  61. package/dist/SchemaUtils.js +444 -12
  62. package/dist/SchemaUtils.js.map +1 -1
  63. package/dist/SecurityUtils.d.ts +55 -8
  64. package/dist/SecurityUtils.js +162 -49
  65. package/dist/SecurityUtils.js.map +1 -1
  66. package/dist/ServerlessAdapter.d.ts +156 -7
  67. package/dist/ServerlessAdapter.js +399 -42
  68. package/dist/ServerlessAdapter.js.map +1 -1
  69. package/dist/SessionManager.d.ts +11 -2
  70. package/dist/SessionManager.js +50 -18
  71. package/dist/SessionManager.js.map +1 -1
  72. package/dist/SslConfig.d.ts +2 -1
  73. package/dist/SslConfig.js +4 -1
  74. package/dist/SslConfig.js.map +1 -1
  75. package/dist/SwaigActions.generated.d.ts +23 -0
  76. package/dist/SwaigActions.generated.js +8 -0
  77. package/dist/SwaigActions.generated.js.map +1 -0
  78. package/dist/SwaigContracts.d.ts +15 -0
  79. package/dist/SwaigContracts.generated.d.ts +264 -0
  80. package/dist/SwaigContracts.generated.js +12 -0
  81. package/dist/SwaigContracts.generated.js.map +1 -0
  82. package/dist/SwaigContracts.js +2 -0
  83. package/dist/SwaigContracts.js.map +1 -0
  84. package/dist/SwaigFunction.d.ts +110 -4
  85. package/dist/SwaigFunction.js +151 -19
  86. package/dist/SwaigFunction.js.map +1 -1
  87. package/dist/SwmlBuilder.d.ts +68 -19
  88. package/dist/SwmlBuilder.js +74 -38
  89. package/dist/SwmlBuilder.js.map +1 -1
  90. package/dist/SwmlVerbMethods.generated.d.ts +41 -4
  91. package/dist/SwmlVerbMethods.generated.js +1 -1
  92. package/dist/TypeInference.d.ts +39 -3
  93. package/dist/TypeInference.js +256 -16
  94. package/dist/TypeInference.js.map +1 -1
  95. package/dist/WebService.d.ts +79 -11
  96. package/dist/WebService.js +320 -109
  97. package/dist/WebService.js.map +1 -1
  98. package/dist/WebhookMiddleware.d.ts +119 -0
  99. package/dist/WebhookMiddleware.js +271 -0
  100. package/dist/WebhookMiddleware.js.map +1 -0
  101. package/dist/WebhookValidator.d.ts +92 -0
  102. package/dist/WebhookValidator.js +409 -0
  103. package/dist/WebhookValidator.js.map +1 -0
  104. package/dist/agents/BedrockAgent.d.ts +162 -0
  105. package/dist/agents/BedrockAgent.js +323 -0
  106. package/dist/agents/BedrockAgent.js.map +1 -0
  107. package/dist/ai-chat/AIChatClient.d.ts +316 -0
  108. package/dist/ai-chat/AIChatClient.js +535 -0
  109. package/dist/ai-chat/AIChatClient.js.map +1 -0
  110. package/dist/ai-chat/ChatGateway.d.ts +408 -0
  111. package/dist/ai-chat/ChatGateway.js +747 -0
  112. package/dist/ai-chat/ChatGateway.js.map +1 -0
  113. package/dist/ai-chat/HandoffRouter.d.ts +282 -0
  114. package/dist/ai-chat/HandoffRouter.js +518 -0
  115. package/dist/ai-chat/HandoffRouter.js.map +1 -0
  116. package/dist/ai-chat/index.d.ts +12 -0
  117. package/dist/ai-chat/index.js +10 -0
  118. package/dist/ai-chat/index.js.map +1 -0
  119. package/dist/capabilities.d.ts +64 -0
  120. package/dist/capabilities.js +95 -0
  121. package/dist/capabilities.js.map +1 -0
  122. package/dist/cli/agent-loader.d.ts +32 -6
  123. package/dist/cli/agent-loader.js +95 -34
  124. package/dist/cli/agent-loader.js.map +1 -1
  125. package/dist/cli/datamap-exec.d.ts +175 -0
  126. package/dist/cli/datamap-exec.js +1284 -0
  127. package/dist/cli/datamap-exec.js.map +1 -0
  128. package/dist/cli/function-args.d.ts +40 -0
  129. package/dist/cli/function-args.js +146 -0
  130. package/dist/cli/function-args.js.map +1 -0
  131. package/dist/cli/simulation.d.ts +99 -0
  132. package/dist/cli/simulation.js +490 -0
  133. package/dist/cli/simulation.js.map +1 -0
  134. package/dist/cli/swaig-test.d.ts +12 -25
  135. package/dist/cli/swaig-test.js +935 -367
  136. package/dist/cli/swaig-test.js.map +1 -1
  137. package/dist/cli/tsdocs/agentsNote.d.ts +30 -0
  138. package/dist/cli/tsdocs/agentsNote.js +92 -0
  139. package/dist/cli/tsdocs/agentsNote.js.map +1 -0
  140. package/dist/cli/tsdocs/api-index.json +1 -0
  141. package/dist/cli/tsdocs/api.d.ts +91 -0
  142. package/dist/cli/tsdocs/api.js +372 -0
  143. package/dist/cli/tsdocs/api.js.map +1 -0
  144. package/dist/cli/tsdocs/apiTypes.d.ts +64 -0
  145. package/dist/cli/tsdocs/apiTypes.js +7 -0
  146. package/dist/cli/tsdocs/apiTypes.js.map +1 -0
  147. package/dist/cli/tsdocs/bin.d.ts +6 -0
  148. package/dist/cli/tsdocs/bin.js +23 -0
  149. package/dist/cli/tsdocs/bin.js.map +1 -0
  150. package/dist/cli/tsdocs/files.d.ts +108 -0
  151. package/dist/cli/tsdocs/files.js +287 -0
  152. package/dist/cli/tsdocs/files.js.map +1 -0
  153. package/dist/cli/tsdocs/index.d.ts +28 -0
  154. package/dist/cli/tsdocs/index.js +307 -0
  155. package/dist/cli/tsdocs/index.js.map +1 -0
  156. package/dist/cli/tsdocs/live.d.ts +41 -0
  157. package/dist/cli/tsdocs/live.js +202 -0
  158. package/dist/cli/tsdocs/live.js.map +1 -0
  159. package/dist/cli/tsdocs/locate.d.ts +51 -0
  160. package/dist/cli/tsdocs/locate.js +75 -0
  161. package/dist/cli/tsdocs/locate.js.map +1 -0
  162. package/dist/cli/tsdocs/render.d.ts +75 -0
  163. package/dist/cli/tsdocs/render.js +323 -0
  164. package/dist/cli/tsdocs/render.js.map +1 -0
  165. package/dist/cli/tsdocs/topics.d.ts +43 -0
  166. package/dist/cli/tsdocs/topics.js +916 -0
  167. package/dist/cli/tsdocs/topics.js.map +1 -0
  168. package/dist/index.d.ts +102 -39
  169. package/dist/index.js +107 -30
  170. package/dist/index.js.map +1 -1
  171. package/dist/livewire/index.d.ts +229 -155
  172. package/dist/livewire/index.js +298 -195
  173. package/dist/livewire/index.js.map +1 -1
  174. package/dist/llms.txt +65 -0
  175. package/dist/prefabs/CallSessionStore.d.ts +52 -0
  176. package/dist/prefabs/CallSessionStore.js +72 -0
  177. package/dist/prefabs/CallSessionStore.js.map +1 -0
  178. package/dist/prefabs/ConciergeAgent.d.ts +13 -1
  179. package/dist/prefabs/ConciergeAgent.js +36 -31
  180. package/dist/prefabs/ConciergeAgent.js.map +1 -1
  181. package/dist/prefabs/FAQBotAgent.d.ts +9 -1
  182. package/dist/prefabs/FAQBotAgent.js +113 -47
  183. package/dist/prefabs/FAQBotAgent.js.map +1 -1
  184. package/dist/prefabs/InfoGathererAgent.d.ts +22 -3
  185. package/dist/prefabs/InfoGathererAgent.js +67 -58
  186. package/dist/prefabs/InfoGathererAgent.js.map +1 -1
  187. package/dist/prefabs/ReceptionistAgent.d.ts +16 -5
  188. package/dist/prefabs/ReceptionistAgent.js +39 -31
  189. package/dist/prefabs/ReceptionistAgent.js.map +1 -1
  190. package/dist/prefabs/SurveyAgent.d.ts +22 -4
  191. package/dist/prefabs/SurveyAgent.js +84 -68
  192. package/dist/prefabs/SurveyAgent.js.map +1 -1
  193. package/dist/relay/Action.d.ts +44 -21
  194. package/dist/relay/Action.js +75 -17
  195. package/dist/relay/Action.js.map +1 -1
  196. package/dist/relay/Call.d.ts +312 -47
  197. package/dist/relay/Call.js +369 -18
  198. package/dist/relay/Call.js.map +1 -1
  199. package/dist/relay/Deferred.js.map +1 -1
  200. package/dist/relay/Message.d.ts +26 -5
  201. package/dist/relay/Message.js +36 -6
  202. package/dist/relay/Message.js.map +1 -1
  203. package/dist/relay/RelayClient.d.ts +79 -14
  204. package/dist/relay/RelayClient.js +268 -61
  205. package/dist/relay/RelayClient.js.map +1 -1
  206. package/dist/relay/RelayError.d.ts +14 -0
  207. package/dist/relay/RelayError.js +15 -0
  208. package/dist/relay/RelayError.js.map +1 -1
  209. package/dist/relay/RelayEvent.d.ts +96 -65
  210. package/dist/relay/RelayEvent.js +85 -56
  211. package/dist/relay/RelayEvent.js.map +1 -1
  212. package/dist/relay/closedSets.d.ts +111 -0
  213. package/dist/relay/closedSets.js +83 -0
  214. package/dist/relay/closedSets.js.map +1 -0
  215. package/dist/relay/index.d.ts +3 -1
  216. package/dist/relay/index.js +1 -0
  217. package/dist/relay/index.js.map +1 -1
  218. package/dist/relay/normalize.js.map +1 -1
  219. package/dist/relay/protocol.types.generated.d.ts +1195 -0
  220. package/dist/relay/protocol.types.generated.js +8 -0
  221. package/dist/relay/protocol.types.generated.js.map +1 -0
  222. package/dist/relay/types.d.ts +81 -39
  223. package/dist/rest/HttpClient.d.ts +36 -8
  224. package/dist/rest/HttpClient.js +233 -40
  225. package/dist/rest/HttpClient.js.map +1 -1
  226. package/dist/rest/RequestOptions.d.ts +110 -0
  227. package/dist/rest/RequestOptions.js +113 -0
  228. package/dist/rest/RequestOptions.js.map +1 -0
  229. package/dist/rest/RestError.d.ts +52 -6
  230. package/dist/rest/RestError.js +89 -5
  231. package/dist/rest/RestError.js.map +1 -1
  232. package/dist/rest/base/CrudResource.d.ts +16 -26
  233. package/dist/rest/base/CrudResource.js +19 -33
  234. package/dist/rest/base/CrudResource.js.map +1 -1
  235. package/dist/rest/base/CrudWithAddresses.d.ts +3 -2
  236. package/dist/rest/base/CrudWithAddresses.js +2 -2
  237. package/dist/rest/base/CrudWithAddresses.js.map +1 -1
  238. package/dist/rest/base/FabricResource.d.ts +15 -0
  239. package/dist/rest/base/FabricResource.js +16 -0
  240. package/dist/rest/base/FabricResource.js.map +1 -0
  241. package/dist/rest/base/ReadResource.d.ts +62 -0
  242. package/dist/rest/base/ReadResource.js +69 -0
  243. package/dist/rest/base/ReadResource.js.map +1 -0
  244. package/dist/rest/callHandler.d.ts +63 -0
  245. package/dist/rest/callHandler.js +62 -0
  246. package/dist/rest/callHandler.js.map +1 -0
  247. package/dist/rest/index.d.ts +19 -76
  248. package/dist/rest/index.js +49 -120
  249. package/dist/rest/index.js.map +1 -1
  250. package/dist/rest/namespaces/_client_tree_generated.d.ts +103 -0
  251. package/dist/rest/namespaces/_client_tree_generated.js +171 -0
  252. package/dist/rest/namespaces/_client_tree_generated.js.map +1 -0
  253. package/dist/rest/namespaces/addresses.d.ts +3 -42
  254. package/dist/rest/namespaces/addresses.js +3 -50
  255. package/dist/rest/namespaces/addresses.js.map +1 -1
  256. package/dist/rest/namespaces/calling.d.ts +6 -378
  257. package/dist/rest/namespaces/calling.js +6 -473
  258. package/dist/rest/namespaces/calling.js.map +1 -1
  259. package/dist/rest/namespaces/calling.resources.generated.d.ts +165 -0
  260. package/dist/rest/namespaces/calling.resources.generated.js +477 -0
  261. package/dist/rest/namespaces/calling.resources.generated.js.map +1 -0
  262. package/dist/rest/namespaces/calling.types.generated.d.ts +2917 -0
  263. package/dist/rest/namespaces/calling.types.generated.js +8 -0
  264. package/dist/rest/namespaces/calling.types.generated.js.map +1 -0
  265. package/dist/rest/namespaces/chat.d.ts +4 -18
  266. package/dist/rest/namespaces/chat.js +4 -21
  267. package/dist/rest/namespaces/chat.js.map +1 -1
  268. package/dist/rest/namespaces/chat.resources.generated.d.ts +12 -0
  269. package/dist/rest/namespaces/chat.resources.generated.js +29 -0
  270. package/dist/rest/namespaces/chat.resources.generated.js.map +1 -0
  271. package/dist/rest/namespaces/chat.types.generated.d.ts +56 -0
  272. package/dist/rest/namespaces/chat.types.generated.js +8 -0
  273. package/dist/rest/namespaces/chat.types.generated.js.map +1 -0
  274. package/dist/rest/namespaces/datasphere.d.ts +6 -62
  275. package/dist/rest/namespaces/datasphere.js +6 -72
  276. package/dist/rest/namespaces/datasphere.js.map +1 -1
  277. package/dist/rest/namespaces/datasphere.resources.generated.d.ts +25 -0
  278. package/dist/rest/namespaces/datasphere.resources.generated.js +50 -0
  279. package/dist/rest/namespaces/datasphere.resources.generated.js.map +1 -0
  280. package/dist/rest/namespaces/datasphere.types.generated.d.ts +180 -0
  281. package/dist/rest/namespaces/datasphere.types.generated.js +8 -0
  282. package/dist/rest/namespaces/datasphere.types.generated.js.map +1 -0
  283. package/dist/rest/namespaces/fabric.d.ts +11 -273
  284. package/dist/rest/namespaces/fabric.js +14 -363
  285. package/dist/rest/namespaces/fabric.js.map +1 -1
  286. package/dist/rest/namespaces/fabric.resources.generated.d.ts +195 -0
  287. package/dist/rest/namespaces/fabric.resources.generated.js +399 -0
  288. package/dist/rest/namespaces/fabric.resources.generated.js.map +1 -0
  289. package/dist/rest/namespaces/fabric.types.generated.d.ts +4557 -0
  290. package/dist/rest/namespaces/fabric.types.generated.js +8 -0
  291. package/dist/rest/namespaces/fabric.types.generated.js.map +1 -0
  292. package/dist/rest/namespaces/fax.resources.generated.d.ts +6 -0
  293. package/dist/rest/namespaces/fax.resources.generated.js +14 -0
  294. package/dist/rest/namespaces/fax.resources.generated.js.map +1 -0
  295. package/dist/rest/namespaces/fax.types.generated.d.ts +131 -0
  296. package/dist/rest/namespaces/fax.types.generated.js +8 -0
  297. package/dist/rest/namespaces/fax.types.generated.js.map +1 -0
  298. package/dist/rest/namespaces/imported-numbers.d.ts +3 -18
  299. package/dist/rest/namespaces/imported-numbers.js +3 -21
  300. package/dist/rest/namespaces/imported-numbers.js.map +1 -1
  301. package/dist/rest/namespaces/logs.d.ts +10 -101
  302. package/dist/rest/namespaces/logs.js +10 -128
  303. package/dist/rest/namespaces/logs.js.map +1 -1
  304. package/dist/rest/namespaces/logs.resources.generated.d.ts +9 -0
  305. package/dist/rest/namespaces/logs.resources.generated.js +17 -0
  306. package/dist/rest/namespaces/logs.resources.generated.js.map +1 -0
  307. package/dist/rest/namespaces/logs.types.generated.d.ts +131 -0
  308. package/dist/rest/namespaces/logs.types.generated.js +8 -0
  309. package/dist/rest/namespaces/logs.types.generated.js.map +1 -0
  310. package/dist/rest/namespaces/lookup.d.ts +3 -25
  311. package/dist/rest/namespaces/lookup.js +3 -27
  312. package/dist/rest/namespaces/lookup.js.map +1 -1
  313. package/dist/rest/namespaces/message.resources.generated.d.ts +6 -0
  314. package/dist/rest/namespaces/message.resources.generated.js +14 -0
  315. package/dist/rest/namespaces/message.resources.generated.js.map +1 -0
  316. package/dist/rest/namespaces/message.types.generated.d.ts +122 -0
  317. package/dist/rest/namespaces/message.types.generated.js +8 -0
  318. package/dist/rest/namespaces/message.types.generated.js.map +1 -0
  319. package/dist/rest/namespaces/messages.resources.generated.d.ts +18 -0
  320. package/dist/rest/namespaces/messages.resources.generated.js +44 -0
  321. package/dist/rest/namespaces/messages.resources.generated.js.map +1 -0
  322. package/dist/rest/namespaces/messages.types.generated.d.ts +99 -0
  323. package/dist/rest/namespaces/messages.types.generated.js +8 -0
  324. package/dist/rest/namespaces/messages.types.generated.js.map +1 -0
  325. package/dist/rest/namespaces/mfa.d.ts +3 -42
  326. package/dist/rest/namespaces/mfa.js +3 -49
  327. package/dist/rest/namespaces/mfa.js.map +1 -1
  328. package/dist/rest/namespaces/number-groups.d.ts +3 -45
  329. package/dist/rest/namespaces/number-groups.js +3 -53
  330. package/dist/rest/namespaces/number-groups.js.map +1 -1
  331. package/dist/rest/namespaces/phone-numbers.d.ts +9 -57
  332. package/dist/rest/namespaces/phone-numbers.js +9 -63
  333. package/dist/rest/namespaces/phone-numbers.js.map +1 -1
  334. package/dist/rest/namespaces/project.d.ts +6 -42
  335. package/dist/rest/namespaces/project.js +6 -51
  336. package/dist/rest/namespaces/project.js.map +1 -1
  337. package/dist/rest/namespaces/project.resources.generated.d.ts +17 -0
  338. package/dist/rest/namespaces/project.resources.generated.js +44 -0
  339. package/dist/rest/namespaces/project.resources.generated.js.map +1 -0
  340. package/dist/rest/namespaces/project.types.generated.d.ts +58 -0
  341. package/dist/rest/namespaces/project.types.generated.js +8 -0
  342. package/dist/rest/namespaces/project.types.generated.js.map +1 -0
  343. package/dist/rest/namespaces/projects.resources.generated.d.ts +12 -0
  344. package/dist/rest/namespaces/projects.resources.generated.js +25 -0
  345. package/dist/rest/namespaces/projects.resources.generated.js.map +1 -0
  346. package/dist/rest/namespaces/projects.types.generated.d.ts +77 -0
  347. package/dist/rest/namespaces/projects.types.generated.js +8 -0
  348. package/dist/rest/namespaces/projects.types.generated.js.map +1 -0
  349. package/dist/rest/namespaces/pubsub.d.ts +4 -18
  350. package/dist/rest/namespaces/pubsub.js +4 -21
  351. package/dist/rest/namespaces/pubsub.js.map +1 -1
  352. package/dist/rest/namespaces/pubsub.resources.generated.d.ts +12 -0
  353. package/dist/rest/namespaces/pubsub.resources.generated.js +29 -0
  354. package/dist/rest/namespaces/pubsub.resources.generated.js.map +1 -0
  355. package/dist/rest/namespaces/pubsub.types.generated.d.ts +60 -0
  356. package/dist/rest/namespaces/pubsub.types.generated.js +8 -0
  357. package/dist/rest/namespaces/pubsub.types.generated.js.map +1 -0
  358. package/dist/rest/namespaces/queues.d.ts +3 -38
  359. package/dist/rest/namespaces/queues.js +3 -44
  360. package/dist/rest/namespaces/queues.js.map +1 -1
  361. package/dist/rest/namespaces/recordings.d.ts +3 -34
  362. package/dist/rest/namespaces/recordings.js +3 -40
  363. package/dist/rest/namespaces/recordings.js.map +1 -1
  364. package/dist/rest/namespaces/registry.d.ts +6 -137
  365. package/dist/rest/namespaces/registry.js +6 -173
  366. package/dist/rest/namespaces/registry.js.map +1 -1
  367. package/dist/rest/namespaces/relay-rest.resources.generated.d.ts +168 -0
  368. package/dist/rest/namespaces/relay-rest.resources.generated.js +430 -0
  369. package/dist/rest/namespaces/relay-rest.resources.generated.js.map +1 -0
  370. package/dist/rest/namespaces/relay-rest.types.generated.d.ts +1906 -0
  371. package/dist/rest/namespaces/relay-rest.types.generated.js +8 -0
  372. package/dist/rest/namespaces/relay-rest.types.generated.js.map +1 -0
  373. package/dist/rest/namespaces/short-codes.d.ts +3 -35
  374. package/dist/rest/namespaces/short-codes.js +3 -41
  375. package/dist/rest/namespaces/short-codes.js.map +1 -1
  376. package/dist/rest/namespaces/sip-profile.d.ts +3 -24
  377. package/dist/rest/namespaces/sip-profile.js +3 -29
  378. package/dist/rest/namespaces/sip-profile.js.map +1 -1
  379. package/dist/rest/namespaces/verified-callers.d.ts +3 -29
  380. package/dist/rest/namespaces/verified-callers.js +3 -34
  381. package/dist/rest/namespaces/verified-callers.js.map +1 -1
  382. package/dist/rest/namespaces/video.d.ts +10 -227
  383. package/dist/rest/namespaces/video.js +10 -288
  384. package/dist/rest/namespaces/video.js.map +1 -1
  385. package/dist/rest/namespaces/video.resources.generated.d.ts +81 -0
  386. package/dist/rest/namespaces/video.resources.generated.js +171 -0
  387. package/dist/rest/namespaces/video.resources.generated.js.map +1 -0
  388. package/dist/rest/namespaces/video.types.generated.d.ts +771 -0
  389. package/dist/rest/namespaces/video.types.generated.js +8 -0
  390. package/dist/rest/namespaces/video.types.generated.js.map +1 -0
  391. package/dist/rest/namespaces/voice.resources.generated.d.ts +9 -0
  392. package/dist/rest/namespaces/voice.resources.generated.js +17 -0
  393. package/dist/rest/namespaces/voice.resources.generated.js.map +1 -0
  394. package/dist/rest/namespaces/voice.types.generated.d.ts +214 -0
  395. package/dist/rest/namespaces/voice.types.generated.js +8 -0
  396. package/dist/rest/namespaces/voice.types.generated.js.map +1 -0
  397. package/dist/rest/pagination.d.ts +5 -1
  398. package/dist/rest/pagination.js +40 -20
  399. package/dist/rest/pagination.js.map +1 -1
  400. package/dist/rest/types.d.ts +21 -3
  401. package/dist/schema.json +299 -98
  402. package/dist/skills/SkillBase.d.ts +97 -20
  403. package/dist/skills/SkillBase.js +100 -17
  404. package/dist/skills/SkillBase.js.map +1 -1
  405. package/dist/skills/SkillManager.d.ts +66 -13
  406. package/dist/skills/SkillManager.js +101 -20
  407. package/dist/skills/SkillManager.js.map +1 -1
  408. package/dist/skills/SkillName.d.ts +44 -0
  409. package/dist/skills/SkillName.js +2 -0
  410. package/dist/skills/SkillName.js.map +1 -0
  411. package/dist/skills/SkillRegistry.d.ts +47 -7
  412. package/dist/skills/SkillRegistry.js +85 -13
  413. package/dist/skills/SkillRegistry.js.map +1 -1
  414. package/dist/skills/builtin/api_ninjas_trivia.d.ts +3 -1
  415. package/dist/skills/builtin/api_ninjas_trivia.js +24 -8
  416. package/dist/skills/builtin/api_ninjas_trivia.js.map +1 -1
  417. package/dist/skills/builtin/ask_claude.d.ts +14 -4
  418. package/dist/skills/builtin/ask_claude.js +36 -14
  419. package/dist/skills/builtin/ask_claude.js.map +1 -1
  420. package/dist/skills/builtin/claude_skills.d.ts +3 -1
  421. package/dist/skills/builtin/claude_skills.js +19 -15
  422. package/dist/skills/builtin/claude_skills.js.map +1 -1
  423. package/dist/skills/builtin/custom_skills.d.ts +10 -3
  424. package/dist/skills/builtin/custom_skills.js +26 -9
  425. package/dist/skills/builtin/custom_skills.js.map +1 -1
  426. package/dist/skills/builtin/datasphere.d.ts +5 -3
  427. package/dist/skills/builtin/datasphere.js +34 -25
  428. package/dist/skills/builtin/datasphere.js.map +1 -1
  429. package/dist/skills/builtin/datasphere_serverless.d.ts +12 -1
  430. package/dist/skills/builtin/datasphere_serverless.js +17 -6
  431. package/dist/skills/builtin/datasphere_serverless.js.map +1 -1
  432. package/dist/skills/builtin/datetime.d.ts +3 -1
  433. package/dist/skills/builtin/datetime.js +11 -7
  434. package/dist/skills/builtin/datetime.js.map +1 -1
  435. package/dist/skills/builtin/google_maps.d.ts +29 -14
  436. package/dist/skills/builtin/google_maps.js +89 -269
  437. package/dist/skills/builtin/google_maps.js.map +1 -1
  438. package/dist/skills/builtin/index.d.ts +8 -6
  439. package/dist/skills/builtin/index.js +8 -6
  440. package/dist/skills/builtin/index.js.map +1 -1
  441. package/dist/skills/builtin/info_gatherer.d.ts +4 -2
  442. package/dist/skills/builtin/info_gatherer.js +20 -12
  443. package/dist/skills/builtin/info_gatherer.js.map +1 -1
  444. package/dist/skills/builtin/joke.d.ts +16 -6
  445. package/dist/skills/builtin/joke.js +54 -37
  446. package/dist/skills/builtin/joke.js.map +1 -1
  447. package/dist/skills/builtin/math.d.ts +3 -1
  448. package/dist/skills/builtin/math.js +14 -9
  449. package/dist/skills/builtin/math.js.map +1 -1
  450. package/dist/skills/builtin/mcp_gateway.d.ts +34 -5
  451. package/dist/skills/builtin/mcp_gateway.js +77 -46
  452. package/dist/skills/builtin/mcp_gateway.js.map +1 -1
  453. package/dist/skills/builtin/native_vector_search.d.ts +19 -5
  454. package/dist/skills/builtin/native_vector_search.js +179 -61
  455. package/dist/skills/builtin/native_vector_search.js.map +1 -1
  456. package/dist/skills/builtin/play_background_file.d.ts +21 -15
  457. package/dist/skills/builtin/play_background_file.js +43 -122
  458. package/dist/skills/builtin/play_background_file.js.map +1 -1
  459. package/dist/skills/builtin/spider.d.ts +46 -1
  460. package/dist/skills/builtin/spider.js +317 -101
  461. package/dist/skills/builtin/spider.js.map +1 -1
  462. package/dist/skills/builtin/swml_transfer.d.ts +14 -3
  463. package/dist/skills/builtin/swml_transfer.js +45 -24
  464. package/dist/skills/builtin/swml_transfer.js.map +1 -1
  465. package/dist/skills/builtin/weather_api.d.ts +24 -13
  466. package/dist/skills/builtin/weather_api.js +57 -31
  467. package/dist/skills/builtin/weather_api.js.map +1 -1
  468. package/dist/skills/builtin/web_search.d.ts +48 -10
  469. package/dist/skills/builtin/web_search.js +301 -84
  470. package/dist/skills/builtin/web_search.js.map +1 -1
  471. package/dist/skills/builtin/wikipedia_search.d.ts +4 -2
  472. package/dist/skills/builtin/wikipedia_search.js +32 -14
  473. package/dist/skills/builtin/wikipedia_search.js.map +1 -1
  474. package/dist/skills/index.d.ts +3 -2
  475. package/dist/skills/index.js +1 -1
  476. package/dist/skills/index.js.map +1 -1
  477. package/dist/swml_verbs_generated.d.ts +2824 -0
  478. package/dist/swml_verbs_generated.js +12 -0
  479. package/dist/swml_verbs_generated.js.map +1 -0
  480. package/dist/types.d.ts +65 -6
  481. package/dist/web.d.ts +27 -0
  482. package/dist/web.js +20 -0
  483. package/dist/web.js.map +1 -0
  484. package/docs/agent-guide.md +1034 -0
  485. package/docs/ai_chat.md +560 -0
  486. package/docs/api-reference.md +3534 -0
  487. package/docs/architecture.md +846 -0
  488. package/docs/bedrock_agent.md +411 -0
  489. package/docs/cli-guide.md +403 -0
  490. package/docs/cloud_functions_guide.md +399 -0
  491. package/docs/configuration.md +548 -0
  492. package/docs/contexts-guide.md +884 -0
  493. package/docs/datamap-guide.md +1160 -0
  494. package/docs/developer_pain_points.md +548 -0
  495. package/docs/llm_parameters.md +298 -0
  496. package/docs/mcp_gateway_reference.md +354 -0
  497. package/docs/mcp_integration.md +162 -0
  498. package/docs/pgi_agent_guide.md +1820 -0
  499. package/docs/prefabs-guide.md +815 -0
  500. package/docs/programmatically_governed_inference.md +301 -0
  501. package/docs/sdk_features.md +501 -0
  502. package/docs/security.md +559 -0
  503. package/docs/serverless-guide.md +520 -0
  504. package/docs/skills-guide.md +1223 -0
  505. package/docs/skills-system.md +405 -0
  506. package/docs/skills_parameter_schema.md +417 -0
  507. package/docs/swaig-reference.md +1938 -0
  508. package/docs/swml_service_guide.md +573 -0
  509. package/docs/third_party_skills.md +410 -0
  510. package/docs/web_service.md +666 -0
  511. package/examples/README.md +202 -0
  512. package/examples/advanced-datamap.ts +108 -0
  513. package/examples/advanced-dynamic-config.ts +70 -0
  514. package/examples/auto-vivified.ts +95 -0
  515. package/examples/call-flow.ts +65 -0
  516. package/examples/comprehensive-dynamic.ts +125 -0
  517. package/examples/contexts-steps.ts +65 -0
  518. package/examples/custom-path.ts +45 -0
  519. package/examples/datamap-tools.ts +51 -0
  520. package/examples/datasphere-multi-instance.ts +66 -0
  521. package/examples/datasphere-serverless-env.ts +87 -0
  522. package/examples/datasphere-webhook-env.ts +65 -0
  523. package/examples/datasphere.ts +56 -0
  524. package/examples/declarative.ts +100 -0
  525. package/examples/dynamic-config.ts +54 -0
  526. package/examples/dynamic-info-gatherer.ts +69 -0
  527. package/examples/dynamic-swml-service.ts +67 -0
  528. package/examples/gather-info.ts +95 -0
  529. package/examples/gather-per-question-functions-demo.ts +139 -0
  530. package/examples/joke-agent.ts +34 -0
  531. package/examples/kubernetes-agent.ts +45 -0
  532. package/examples/llm-params.ts +75 -0
  533. package/examples/mcp-agent.ts +89 -0
  534. package/examples/mcp-gateway.ts +81 -0
  535. package/examples/multi-agent.ts +80 -0
  536. package/examples/multi-endpoint.ts +79 -0
  537. package/examples/pgi/case-domain.ts +114 -0
  538. package/examples/pgi/serve.ts +16 -0
  539. package/examples/pgi/support-agent.ts +265 -0
  540. package/examples/pom-prompt.ts +70 -0
  541. package/examples/prefab-concierge.ts +42 -0
  542. package/examples/prefab-faq.ts +58 -0
  543. package/examples/prefab-info-gatherer.ts +33 -0
  544. package/examples/prefab-receptionist.ts +36 -0
  545. package/examples/prefab-survey.ts +69 -0
  546. package/examples/quickstart-agent.ts +29 -0
  547. package/examples/quickstart-relay.ts +25 -0
  548. package/examples/quickstart-rest.ts +25 -0
  549. package/examples/record-call.ts +65 -0
  550. package/examples/relay-demo.ts +33 -0
  551. package/examples/room-and-sip.ts +93 -0
  552. package/examples/serverless-lambda.ts +43 -0
  553. package/examples/session-state.ts +83 -0
  554. package/examples/simple-agent.ts +36 -0
  555. package/examples/simple-static.ts +29 -0
  556. package/examples/skills-demo.ts +41 -0
  557. package/examples/step-function-inheritance-demo.ts +124 -0
  558. package/examples/swaig-features.ts +193 -0
  559. package/examples/swml-service-routing.ts +66 -0
  560. package/examples/swml-service.ts +38 -0
  561. package/examples/swmlservice_ai_sidecar.ts +104 -0
  562. package/examples/swmlservice_swaig_standalone.ts +79 -0
  563. package/examples/tap.ts +67 -0
  564. package/examples/typed-tools.ts +68 -0
  565. package/examples/verb-methods.ts +45 -0
  566. package/examples/web-search-multi-instance.ts +75 -0
  567. package/examples/web-search.ts +38 -0
  568. package/examples/wikipedia.ts +36 -0
  569. package/livewire/README.md +164 -0
  570. package/livewire/docs/migration-guide.md +251 -0
  571. package/livewire/examples/livewire-basic-agent.ts +74 -0
  572. package/livewire/examples/livewire-handoff.ts +196 -0
  573. package/livewire/examples/livewire-multi-tool.ts +121 -0
  574. package/package.json +88 -13
  575. package/relay/README.md +101 -0
  576. package/relay/docs/call-methods.md +622 -0
  577. package/relay/docs/client-reference.md +281 -0
  578. package/relay/docs/events.md +193 -0
  579. package/relay/docs/getting-started.md +221 -0
  580. package/relay/docs/guide.md +386 -0
  581. package/relay/docs/messaging.md +246 -0
  582. package/relay/examples/relay-inbound.ts +66 -0
  583. package/relay/examples/relay-messaging.ts +74 -0
  584. package/relay/examples/relay-outbound.ts +76 -0
  585. package/rest/README.md +141 -0
  586. package/rest/docs/calling.md +313 -0
  587. package/rest/docs/client-reference.md +149 -0
  588. package/rest/docs/fabric.md +192 -0
  589. package/rest/docs/getting-started.md +138 -0
  590. package/rest/docs/guide.md +421 -0
  591. package/rest/docs/namespaces.md +367 -0
  592. package/rest/docs/phone-binding.md +116 -0
  593. package/rest/examples/rest-10dlc-registration.ts +177 -0
  594. package/rest/examples/rest-bind-phone-to-swml-webhook.ts +66 -0
  595. package/rest/examples/rest-calling-ivr-and-ai.ts +136 -0
  596. package/rest/examples/rest-calling-play-and-record.ts +98 -0
  597. package/rest/examples/rest-client.ts +59 -0
  598. package/rest/examples/rest-datasphere-search.ts +81 -0
  599. package/rest/examples/rest-fabric-conferences-and-routing.ts +147 -0
  600. package/rest/examples/rest-fabric-subscribers-and-sip.ts +106 -0
  601. package/rest/examples/rest-fabric-swml-and-callflows.ts +101 -0
  602. package/rest/examples/rest-manage-resources.ts +64 -0
  603. package/rest/examples/rest-phone-number-management.ts +200 -0
  604. package/rest/examples/rest-queues-mfa-and-recordings.ts +143 -0
  605. package/rest/examples/rest-video-rooms.ts +186 -0
  606. package/tutorial/fred/fred.ts +218 -0
  607. package/tutorial/fred/tutorial/01-introduction.md +141 -0
  608. package/tutorial/fred/tutorial/02-setup.md +304 -0
  609. package/tutorial/fred/tutorial/03-basic-agent.md +284 -0
  610. package/tutorial/fred/tutorial/04-wikipedia-skill.md +298 -0
  611. package/tutorial/fred/tutorial/05-custom-functions.md +347 -0
  612. package/tutorial/fred/tutorial/06-running-testing.md +642 -0
  613. package/tutorial/fred/tutorial/README.md +56 -0
  614. package/tutorial/fred/tutorial/appendix-complete-code.md +632 -0
  615. package/tutorial/fred/tutorial/appendix-docker-deployment.md +504 -0
  616. package/tutorial/full-guardrails-agent/handlers.ts +436 -0
  617. package/tutorial/full-guardrails-agent/penny.ts +305 -0
  618. package/tutorial/full-guardrails-agent/reservations.ts +1109 -0
  619. package/tutorial/full-guardrails-agent/tutorial/01-why-guardrails.md +149 -0
  620. package/tutorial/full-guardrails-agent/tutorial/02-design-first.md +139 -0
  621. package/tutorial/full-guardrails-agent/tutorial/03-rules-first.md +422 -0
  622. package/tutorial/full-guardrails-agent/tutorial/04-the-shell.md +238 -0
  623. package/tutorial/full-guardrails-agent/tutorial/05-steps-and-scoping.md +247 -0
  624. package/tutorial/full-guardrails-agent/tutorial/06-tools-that-decide.md +467 -0
  625. package/tutorial/full-guardrails-agent/tutorial/07-gather-and-projection.md +160 -0
  626. package/tutorial/full-guardrails-agent/tutorial/08-the-verification-gate.md +333 -0
  627. package/tutorial/full-guardrails-agent/tutorial/09-people-and-endings.md +313 -0
  628. package/tutorial/full-guardrails-agent/tutorial/10-testing-and-running.md +387 -0
  629. package/tutorial/full-guardrails-agent/tutorial/README.md +106 -0
  630. package/tutorial/full-guardrails-agent/tutorial/appendix-complete-code.md +2207 -0
  631. package/tutorial/full-guardrails-agent/tutorial/appendix-deployment.md +191 -0
  632. package/tutorial/full-guardrails-agent/tutorial/appendix-technique-map.md +117 -0
  633. package/tutorial/full-guardrails-agent/workflow.ts +219 -0
  634. package/tutorial/multi_agents/README.md +175 -0
  635. package/tutorial/multi_agents/advanced_agent.ts +395 -0
  636. package/tutorial/multi_agents/extending_agents.ts +641 -0
  637. package/tutorial/multi_agents/knowledge.ts +59 -0
  638. package/tutorial/multi_agents/lesson1_first_agent.md +368 -0
  639. package/tutorial/multi_agents/lesson2_knowledge_bases.md +404 -0
  640. package/tutorial/multi_agents/lesson3_multi_agent_systems.md +621 -0
  641. package/tutorial/multi_agents/lesson4_advanced_features.md +805 -0
  642. package/tutorial/multi_agents/lesson5_extending_agents.md +856 -0
  643. package/tutorial/multi_agents/pc_builder.ts +529 -0
  644. package/tutorial/multi_agents/sales_agent.ts +88 -0
  645. package/tutorial/multi_agents/sales_agent_with_search.ts +116 -0
  646. package/tutorial/multi_agents/sales_knowledge.md +139 -0
  647. package/tutorial/multi_agents/support_knowledge.md +256 -0
  648. package/dist/cli/mock-data.d.ts +0 -37
  649. package/dist/cli/mock-data.js +0 -80
  650. package/dist/cli/mock-data.js.map +0 -1
  651. package/dist/generateVerbTypes.d.ts +0 -11
  652. package/dist/generateVerbTypes.js +0 -265
  653. package/dist/generateVerbTypes.js.map +0 -1
  654. package/dist/rest/namespaces/compat.d.ts +0 -585
  655. package/dist/rest/namespaces/compat.js +0 -735
  656. package/dist/rest/namespaces/compat.js.map +0 -1
@@ -11,23 +11,16 @@
11
11
  *
12
12
  * Multiple instances are supported; use `tool_name` in config to differentiate.
13
13
  */
14
- import { SkillBase } from '../SkillBase.js';
14
+ import { SkillBase, defineSkillTool } from '../SkillBase.js';
15
15
  import { FunctionResult } from '../../FunctionResult.js';
16
+ import { _publicFetch, _RedirectRefused } from '../../PublicFetch.js';
17
+ import { _RobotsRules } from '../../RobotsTxt.js';
16
18
  import { resolveAndValidateUrl, validateUrl, MAX_SKILL_INPUT_LENGTH } from '../../SecurityUtils.js';
17
19
  import { getLogger } from '../../Logger.js';
18
- import * as cheerio from 'cheerio';
19
20
  const log = getLogger('SpiderSkill');
20
21
  const WHITESPACE_REGEX = /\s+/g;
21
- /** Tags stripped from HTML before text extraction. */
22
- const STRIP_TAG_REGEXES = [
23
- /<script\b[^>]*>[\s\S]*?<\/script>/gi,
24
- /<style\b[^>]*>[\s\S]*?<\/style>/gi,
25
- /<nav\b[^>]*>[\s\S]*?<\/nav>/gi,
26
- /<header\b[^>]*>[\s\S]*?<\/header>/gi,
27
- /<footer\b[^>]*>[\s\S]*?<\/footer>/gi,
28
- /<aside\b[^>]*>[\s\S]*?<\/aside>/gi,
29
- /<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi,
30
- ];
22
+ /** Headers that carry credentials for one origin (as _publicFetch treats them). */
23
+ const ORIGIN_BOUND_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']);
31
24
  /**
32
25
  * Fast web scraping skill optimized for speed and token efficiency.
33
26
  *
@@ -39,7 +32,9 @@ const STRIP_TAG_REGEXES = [
39
32
  *
40
33
  * @example
41
34
  * ```ts
42
- * agent.addSkill('spider', { max_pages: 5, max_depth: 2 });
35
+ * import { AgentBase } from '@signalwire/sdk';
36
+ * const agent = new AgentBase({ name: 'demo', route: '/' });
37
+ * await agent.addSkillByName('spider', { max_pages: 5, max_depth: 2 });
43
38
  * ```
44
39
  */
45
40
  export class SpiderSkill extends SkillBase {
@@ -51,20 +46,64 @@ export class SpiderSkill extends SkillBase {
51
46
  static REQUIRED_PACKAGES = ['cheerio'];
52
47
  static REQUIRED_ENV_VARS = [];
53
48
  static SUPPORTS_MULTIPLE_INSTANCES = true;
49
+ /**
50
+ * Each setting's default. The parameter schema and setup() both read this,
51
+ * so the schema can't advertise a default the skill doesn't use.
52
+ */
53
+ static DEFAULTS = {
54
+ delay: 0.1,
55
+ concurrent_requests: 5,
56
+ timeout: 5,
57
+ max_pages: 1,
58
+ max_depth: 0,
59
+ extract_type: 'fast_text',
60
+ max_text_length: 3000,
61
+ clean_text: true,
62
+ cache_enabled: true,
63
+ follow_robots_txt: false,
64
+ user_agent: 'Spider/1.0 (SignalWire AI Agent)',
65
+ };
66
+ /** The extraction methods scrape_url implements. */
67
+ static EXTRACT_TYPES = [
68
+ 'fast_text',
69
+ 'markdown',
70
+ 'structured',
71
+ ];
72
+ /**
73
+ * Values the schema once listed that were never implemented. They have
74
+ * always worked as fast_text, so they still do, with a warning.
75
+ */
76
+ static LEGACY_EXTRACT_TYPES = new Set([
77
+ 'clean_text',
78
+ 'full_text',
79
+ 'html',
80
+ 'custom',
81
+ ]);
82
+ /** Seconds to keep a site's robots.txt rules; RFC 9309 allows up to 24 hours. */
83
+ static ROBOTS_TTL = 24 * 60 * 60;
54
84
  static getParameterSchema() {
85
+ const d = SpiderSkill.DEFAULTS;
55
86
  return {
56
87
  ...super.getParameterSchema(),
88
+ // The base schema's default is the skill name, but tool_name is a
89
+ // prefix that's empty when unset (Python skill.py:306 too).
90
+ tool_name: {
91
+ type: 'string',
92
+ description: 'Prefix for the tool names, as in <tool_name>_scrape_url, and for the instance key. ' +
93
+ 'Unset, the tools keep their plain names.',
94
+ required: false,
95
+ },
57
96
  delay: {
58
97
  type: 'number',
59
98
  description: 'Delay between requests in seconds',
60
- default: 0.1,
99
+ default: d.delay,
61
100
  required: false,
62
101
  min: 0,
63
102
  },
64
103
  concurrent_requests: {
65
104
  type: 'integer',
66
- description: 'Number of concurrent requests allowed',
67
- default: 5,
105
+ description: 'Deprecated, and has no effect: the spider fetches one page at a time',
106
+ default: d.concurrent_requests,
68
107
  required: false,
69
108
  min: 1,
70
109
  max: 20,
@@ -72,7 +111,7 @@ export class SpiderSkill extends SkillBase {
72
111
  timeout: {
73
112
  type: 'integer',
74
113
  description: 'Request timeout in seconds',
75
- default: 5,
114
+ default: d.timeout,
76
115
  required: false,
77
116
  min: 1,
78
117
  max: 60,
@@ -80,7 +119,7 @@ export class SpiderSkill extends SkillBase {
80
119
  max_pages: {
81
120
  type: 'integer',
82
121
  description: 'Maximum number of pages to scrape',
83
- default: 1,
122
+ default: d.max_pages,
84
123
  required: false,
85
124
  min: 1,
86
125
  max: 100,
@@ -88,7 +127,7 @@ export class SpiderSkill extends SkillBase {
88
127
  max_depth: {
89
128
  type: 'integer',
90
129
  description: 'Maximum crawl depth (0 = single page only)',
91
- default: 0,
130
+ default: d.max_depth,
92
131
  required: false,
93
132
  min: 0,
94
133
  max: 5,
@@ -96,14 +135,14 @@ export class SpiderSkill extends SkillBase {
96
135
  extract_type: {
97
136
  type: 'string',
98
137
  description: 'Content extraction method',
99
- default: 'fast_text',
138
+ default: d.extract_type,
100
139
  required: false,
101
- enum: ['fast_text', 'clean_text', 'full_text', 'html', 'markdown', 'structured', 'custom'],
140
+ enum: [...SpiderSkill.EXTRACT_TYPES],
102
141
  },
103
142
  max_text_length: {
104
143
  type: 'integer',
105
144
  description: 'Maximum text length to return',
106
- default: 3000,
145
+ default: d.max_text_length,
107
146
  required: false,
108
147
  min: 100,
109
148
  max: 100000,
@@ -111,7 +150,7 @@ export class SpiderSkill extends SkillBase {
111
150
  clean_text: {
112
151
  type: 'boolean',
113
152
  description: 'Whether to clean extracted text',
114
- default: true,
153
+ default: d.clean_text,
115
154
  required: false,
116
155
  },
117
156
  selectors: {
@@ -131,7 +170,7 @@ export class SpiderSkill extends SkillBase {
131
170
  user_agent: {
132
171
  type: 'string',
133
172
  description: 'User agent string for requests',
134
- default: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
173
+ default: d.user_agent,
135
174
  required: false,
136
175
  },
137
176
  headers: {
@@ -143,63 +182,72 @@ export class SpiderSkill extends SkillBase {
143
182
  },
144
183
  follow_robots_txt: {
145
184
  type: 'boolean',
146
- // Matches Python's __init__ runtime fallback (skills/spider/skill.py:177
147
- // uses `params.get('follow_robots_txt', False)`). Python's schema said
148
- // True but the runtime and TS both use False — aligning the schema
149
- // eliminates the cross-SDK contract mismatch.
150
- description: 'Whether to respect robots.txt (default: false to match Python runtime)',
151
- default: false,
185
+ description: "Skip pages that the site's robots.txt disallows for user_agent",
186
+ default: d.follow_robots_txt,
152
187
  required: false,
153
188
  },
154
189
  cache_enabled: {
155
190
  type: 'boolean',
156
191
  description: 'Whether to cache scraped pages',
157
- default: true,
192
+ default: d.cache_enabled,
158
193
  required: false,
159
194
  },
160
195
  };
161
196
  }
162
197
  // Runtime state, populated in setup()
163
- delay = 0.1;
164
- concurrentRequests = 5;
165
- timeout = 5;
166
- maxPages = 1;
167
- maxDepth = 0;
168
- extractType = 'fast_text';
169
- maxTextLength = 3000;
170
- cleanText = true;
171
- cacheEnabled = true;
172
- followRobotsTxt = true;
173
- userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
198
+ delay = SpiderSkill.DEFAULTS.delay;
199
+ concurrentRequests = SpiderSkill.DEFAULTS.concurrent_requests;
200
+ timeout = SpiderSkill.DEFAULTS.timeout;
201
+ maxPages = SpiderSkill.DEFAULTS.max_pages;
202
+ maxDepth = SpiderSkill.DEFAULTS.max_depth;
203
+ extractType = SpiderSkill.DEFAULTS.extract_type;
204
+ maxTextLength = SpiderSkill.DEFAULTS.max_text_length;
205
+ cleanText = SpiderSkill.DEFAULTS.clean_text;
206
+ cacheEnabled = SpiderSkill.DEFAULTS.cache_enabled;
207
+ followRobotsTxt = SpiderSkill.DEFAULTS.follow_robots_txt;
208
+ userAgent = SpiderSkill.DEFAULTS.user_agent;
174
209
  headers = {};
175
210
  selectors = {};
176
211
  compiledFollowPatterns = [];
177
212
  cache = null;
178
213
  cacheMaxSize = 100;
214
+ /** robots.txt rules per origin, with when they expire, when follow_robots_txt is on. */
215
+ robots = new Map();
216
+ // Lazily-loaded optional `cheerio` module, populated in setup(). Non-null
217
+ // for the lifetime of an initialized skill (setup() returns false if absent).
218
+ _cheerio;
179
219
  getInstanceKey() {
180
220
  const toolName = this.getConfig('tool_name', this.skillName);
181
221
  return `${this.skillName}_${toolName}`;
182
222
  }
183
223
  async setup() {
224
+ // Load the optional cheerio dependency once. If it isn't installed, the
225
+ // skill cannot scrape — fail setup cleanly (validatePackages() also flags
226
+ // it). Cached on the instance so the sync extract helpers can use it.
227
+ try {
228
+ this._cheerio = await import('cheerio');
229
+ }
230
+ catch {
231
+ log.error('spider: the optional "cheerio" package is required for this skill but is not installed');
232
+ return false;
233
+ }
234
+ const d = SpiderSkill.DEFAULTS;
184
235
  // Performance
185
- this.delay = this.getConfig('delay', 0.1);
186
- this.concurrentRequests = this.getConfig('concurrent_requests', 5);
187
- this.timeout = this.getConfig('timeout', 5);
236
+ this.delay = this.getConfig('delay', d.delay);
237
+ this.concurrentRequests = this.getConfig('concurrent_requests', d.concurrent_requests);
238
+ this.timeout = this.getConfig('timeout', d.timeout);
188
239
  // Crawl limits
189
- this.maxPages = this.getConfig('max_pages', 1);
190
- this.maxDepth = this.getConfig('max_depth', 0);
240
+ this.maxPages = this.getConfig('max_pages', d.max_pages);
241
+ this.maxDepth = this.getConfig('max_depth', d.max_depth);
191
242
  // Content processing
192
- this.extractType = this.getConfig('extract_type', 'fast_text');
193
- // Python has an internal inconsistency (schema: 10000, __init__ fallback:
194
- // 3000). The effective runtime default is 3000 because Python reads via
195
- // `self.params.get('max_text_length', 3000)` and schema defaults are not
196
- // applied. Use 3000 for parity. Follow-up: reconcile Python schema vs init.
197
- this.maxTextLength = this.getConfig('max_text_length', 3000);
198
- this.cleanText = this.getConfig('clean_text', true);
243
+ this.extractType = this.getConfig('extract_type', d.extract_type);
244
+ this.maxTextLength = this.getConfig('max_text_length', d.max_text_length);
245
+ this.cleanText = this.getConfig('clean_text', d.clean_text);
199
246
  // Features
200
- this.cacheEnabled = this.getConfig('cache_enabled', true);
201
- this.followRobotsTxt = this.getConfig('follow_robots_txt', false);
202
- this.userAgent = this.getConfig('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
247
+ this.cacheEnabled = this.getConfig('cache_enabled', d.cache_enabled);
248
+ this.followRobotsTxt = this.getConfig('follow_robots_txt', d.follow_robots_txt);
249
+ this.userAgent = this.getConfig('user_agent', d.user_agent);
250
+ this.robots = new Map();
203
251
  // Optional headers + user-agent merge
204
252
  const headers = this.getConfig('headers', {}) ?? {};
205
253
  this.headers = { ...headers, 'User-Agent': this.userAgent };
@@ -213,7 +261,22 @@ export class SpiderSkill extends SkillBase {
213
261
  return false;
214
262
  }
215
263
  if (this.concurrentRequests < 1 || this.concurrentRequests > 20) {
216
- log.error('spider: concurrent_requests must be between 1 and 20', { concurrent_requests: this.concurrentRequests });
264
+ log.error('spider: concurrent_requests must be between 1 and 20', {
265
+ concurrent_requests: this.concurrentRequests,
266
+ });
267
+ return false;
268
+ }
269
+ if ('concurrent_requests' in this.config) {
270
+ log.warn('spider: concurrent_requests is deprecated and has no effect: the spider fetches one page at a time');
271
+ }
272
+ // Validate the extraction method
273
+ const types = SpiderSkill.EXTRACT_TYPES.join(', ');
274
+ if (SpiderSkill.LEGACY_EXTRACT_TYPES.has(this.extractType)) {
275
+ log.warn(`spider: extract_type '${this.extractType}' was never implemented and works as fast_text; use one of ${types}`);
276
+ this.extractType = 'fast_text';
277
+ }
278
+ else if (!SpiderSkill.EXTRACT_TYPES.includes(this.extractType)) {
279
+ log.error(`spider: unknown extract_type '${this.extractType}'; use one of ${types}`);
217
280
  return false;
218
281
  }
219
282
  if (this.maxPages < 1) {
@@ -267,8 +330,11 @@ export class SpiderSkill extends SkillBase {
267
330
  getTools() {
268
331
  const toolPrefix = this.getConfig('tool_name', '');
269
332
  const prefix = toolPrefix ? `${toolPrefix}_` : '';
333
+ // The three handlers keep their defensive typeof/.trim()/URL-validation
334
+ // narrowing in the private methods: `url`/`start_url` are required, but the
335
+ // model can still emit an empty or wrong-typed value.
270
336
  return [
271
- {
337
+ defineSkillTool({
272
338
  name: `${prefix}scrape_url`,
273
339
  description: 'Extract text content from a single web page',
274
340
  parameters: {
@@ -279,8 +345,8 @@ export class SpiderSkill extends SkillBase {
279
345
  },
280
346
  required: ['url'],
281
347
  handler: async (args, rawData) => this._scrapeUrlHandler(args, rawData),
282
- },
283
- {
348
+ }),
349
+ defineSkillTool({
284
350
  name: `${prefix}crawl_site`,
285
351
  description: 'Crawl multiple pages starting from a URL',
286
352
  parameters: {
@@ -291,8 +357,8 @@ export class SpiderSkill extends SkillBase {
291
357
  },
292
358
  required: ['start_url'],
293
359
  handler: async (args, rawData) => this._crawlSiteHandler(args, rawData),
294
- },
295
- {
360
+ }),
361
+ defineSkillTool({
296
362
  name: `${prefix}extract_structured_data`,
297
363
  description: 'Extract specific data from a web page using selectors',
298
364
  parameters: {
@@ -303,25 +369,168 @@ export class SpiderSkill extends SkillBase {
303
369
  },
304
370
  required: ['url'],
305
371
  handler: async (args, rawData) => this._extractStructuredHandler(args, rawData),
306
- },
372
+ }),
307
373
  ];
308
374
  }
309
375
  // ---------------------------------------------------------------------------
310
376
  // Internal helpers
311
377
  // ---------------------------------------------------------------------------
378
+ /**
379
+ * Redirect a target URL through `SPIDER_BASE_URL` when set. Used by tests
380
+ * to point every outbound fetch at a loopback fixture without requiring
381
+ * the test URL to resolve.
382
+ * Preserves the path (and any query/fragment) from the original target.
383
+ */
384
+ /** Whether a URL is on the operator-set SPIDER_BASE_URL's origin, a trusted target. */
385
+ static _isAuditOrigin(target) {
386
+ const base = process.env['SPIDER_BASE_URL'];
387
+ if (!base)
388
+ return false;
389
+ try {
390
+ return new URL(target).origin === new URL(base).origin;
391
+ }
392
+ catch {
393
+ return false;
394
+ }
395
+ }
396
+ static _redirectForAudit(target) {
397
+ const base = process.env['SPIDER_BASE_URL'];
398
+ if (!base)
399
+ return target;
400
+ try {
401
+ const u = new URL(target);
402
+ // Keep path + search + hash; drop scheme + host.
403
+ const path = `${u.pathname}${u.search}${u.hash}` || '/';
404
+ return `${base.replace(/\/+$/, '')}${path}`;
405
+ }
406
+ catch {
407
+ // Not a parseable URL — assume it's a path already.
408
+ const path = target.startsWith('/') ? target : `/${target}`;
409
+ return `${base.replace(/\/+$/, '')}${path}`;
410
+ }
411
+ }
412
+ /**
413
+ * Whether a URL the caller supplied may be fetched: it must not be private
414
+ * or internal (`SWML_ALLOW_PRIVATE_URLS` allows it). With `SPIDER_BASE_URL`
415
+ * set, the fetch goes to that operator-configured base instead of the URL's
416
+ * host, so the host isn't checked.
417
+ */
418
+ static async _targetAllowed(url) {
419
+ if (SpiderSkill._redirectForAudit(url) !== url)
420
+ return true;
421
+ try {
422
+ await resolveAndValidateUrl(url);
423
+ return true;
424
+ }
425
+ catch {
426
+ return false;
427
+ }
428
+ }
429
+ /**
430
+ * False when follow_robots_txt is on and the site's robots.txt disallows
431
+ * `url` for user_agent.
432
+ *
433
+ * As in Python's `urllib.robotparser`, a robots.txt answered with 401 or
434
+ * 403 disallows everything and any other 4xx allows everything. Rules are
435
+ * kept for {@link ROBOTS_TTL}. A server error or failed request disallows
436
+ * this page without keeping anything, so the next request tries again.
437
+ */
438
+ async _allowedByRobots(url, opts = {}) {
439
+ if (!this.followRobotsTxt)
440
+ return true;
441
+ let origin;
442
+ try {
443
+ // Check the URL as it will be requested: fetch normalizes it, so
444
+ // /public/../private is a request for /private.
445
+ const parsed = new URL(url);
446
+ url = parsed.href;
447
+ origin = parsed.origin;
448
+ }
449
+ catch {
450
+ return false;
451
+ }
452
+ const now = performance.now() / 1000;
453
+ const cached = this.robots.get(origin);
454
+ if (cached && cached.expires > now)
455
+ return cached.rules.canFetch(this.userAgent, url);
456
+ const robotsUrl = `${origin}/robots.txt`;
457
+ const fetchUrl = SpiderSkill._redirectForAudit(robotsUrl);
458
+ // Credentials for the page's origin don't go to another origin's
459
+ // robots.txt, as _publicFetch drops them on a redirect there.
460
+ const headers = { ...this.headers };
461
+ if (opts.withoutCredentials) {
462
+ for (const name of Object.keys(headers)) {
463
+ if (ORIGIN_BOUND_HEADERS.has(name.toLowerCase()))
464
+ delete headers[name];
465
+ }
466
+ }
467
+ let status;
468
+ let body = '';
469
+ const controller = new AbortController();
470
+ const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
471
+ try {
472
+ const response = await _publicFetch(fetchUrl, {
473
+ method: 'GET',
474
+ headers,
475
+ signal: controller.signal,
476
+ allowPrivate: fetchUrl !== robotsUrl || SpiderSkill._isAuditOrigin(robotsUrl),
477
+ });
478
+ status = response.status;
479
+ body = await response.text();
480
+ }
481
+ catch {
482
+ status = 599;
483
+ }
484
+ finally {
485
+ clearTimeout(timer);
486
+ }
487
+ // Unavailable for now: disallow this request, and try again next time.
488
+ if (status >= 500)
489
+ return false;
490
+ let rules;
491
+ if (status === 401 || status === 403)
492
+ rules = _RobotsRules.disallowAll();
493
+ else if (status >= 400)
494
+ rules = _RobotsRules.allowAll();
495
+ else
496
+ rules = new _RobotsRules(body);
497
+ this.robots.set(origin, { rules, expires: now + SpiderSkill.ROBOTS_TTL });
498
+ return rules.canFetch(this.userAgent, url);
499
+ }
312
500
  /** Fetch a URL with caching and timeout handling. Returns null on failure. */
313
501
  async _fetchUrl(url) {
314
502
  if (this.cacheEnabled && this.cache?.has(url)) {
315
503
  log.debug('spider: cache hit', { url });
316
504
  return this.cache.get(url);
317
505
  }
506
+ // The porting-sdk's `audit_skills_dispatch.py` sets `SPIDER_BASE_URL`
507
+ // to redirect every fetch through a loopback fixture. The audit feeds
508
+ // a `https://audit.example/page` target and expects the skill to hit
509
+ // `http://127.0.0.1:NNNN/page` — preserving the path after the host.
510
+ const fetchUrl = SpiderSkill._redirectForAudit(url);
511
+ const startOrigin = new URL(fetchUrl).origin;
512
+ let crossedOrigin = false;
318
513
  const controller = new AbortController();
319
514
  const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
320
515
  try {
321
- const response = await fetch(url, {
516
+ // _publicFetch checks the URL and every redirect, and refuses a
517
+ // connection to a private or internal address. An operator-set
518
+ // SPIDER_BASE_URL is a trusted target, so its fetches skip the check.
519
+ const response = await _publicFetch(fetchUrl, {
322
520
  method: 'GET',
323
521
  headers: this.headers,
324
522
  signal: controller.signal,
523
+ allowPrivate: fetchUrl !== url,
524
+ // Each redirect's target needs its own robots.txt check.
525
+ allowRedirect: this.followRobotsTxt
526
+ ? (target) => {
527
+ // Once a redirect leaves the page's origin, _publicFetch has
528
+ // dropped the credentials for good, and so does robots.txt.
529
+ if (new URL(target).origin !== startOrigin)
530
+ crossedOrigin = true;
531
+ return this._allowedByRobots(target, { withoutCredentials: crossedOrigin });
532
+ }
533
+ : undefined,
325
534
  });
326
535
  if (!response.ok) {
327
536
  log.error('spider: fetch failed', { url, status: response.status });
@@ -344,6 +553,10 @@ export class SpiderSkill extends SkillBase {
344
553
  return cached;
345
554
  }
346
555
  catch (err) {
556
+ if (err instanceof _RedirectRefused) {
557
+ log.info('spider: robots.txt disallows the redirect', { url, target: err.target });
558
+ return null;
559
+ }
347
560
  log.error('spider: fetch error', {
348
561
  url,
349
562
  error: err instanceof Error ? err.message : String(err),
@@ -364,22 +577,30 @@ export class SpiderSkill extends SkillBase {
364
577
  */
365
578
  _fastTextExtract(response) {
366
579
  try {
580
+ // Accept JSON as input too. The porting-sdk audit fixture replies
581
+ // with `Content-Type: application/json` whose body is `{"_raw_html":
582
+ // "<html>…</html>"}`. Unwrap to the inner HTML when that shape is
583
+ // present so the text extractor sees real markup.
584
+ let body = response.body;
585
+ try {
586
+ const parsed = JSON.parse(body);
587
+ if (parsed &&
588
+ typeof parsed === 'object' &&
589
+ typeof parsed['_raw_html'] === 'string') {
590
+ body = parsed['_raw_html'];
591
+ }
592
+ }
593
+ catch {
594
+ // Not JSON; treat as HTML.
595
+ }
367
596
  // Parse through cheerio so every named and numeric HTML entity is
368
597
  // decoded (Python's lxml does this natively). Previously TS only
369
598
  // decoded six hand-coded entities, leaving `&mdash;`, `&#8212;`, etc.
370
599
  // literal in the output.
371
- const $ = cheerio.load(response.body);
600
+ const $ = this._cheerio.load(body);
372
601
  // Strip noise tags before extracting text — matches Python's
373
602
  // lxml drop_tree() on the same 7 tags.
374
- for (const tag of [
375
- 'script',
376
- 'style',
377
- 'nav',
378
- 'header',
379
- 'footer',
380
- 'aside',
381
- 'noscript',
382
- ]) {
603
+ for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside', 'noscript']) {
383
604
  $(tag).remove();
384
605
  }
385
606
  let text = $('body').text();
@@ -395,9 +616,7 @@ export class SpiderSkill extends SkillBase {
395
616
  const keepStart = Math.floor((this.maxTextLength * 2) / 3);
396
617
  const keepEnd = Math.floor(this.maxTextLength / 3);
397
618
  text =
398
- text.slice(0, keepStart) +
399
- '\n\n[...CONTENT TRUNCATED...]\n\n' +
400
- text.slice(-keepEnd);
619
+ text.slice(0, keepStart) + '\n\n[...CONTENT TRUNCATED...]\n\n' + text.slice(-keepEnd);
401
620
  }
402
621
  return text;
403
622
  }
@@ -440,7 +659,7 @@ export class SpiderSkill extends SkillBase {
440
659
  /** Markdown extraction using cheerio. Mirrors Python `_markdown_extract`. */
441
660
  _markdownExtract(response) {
442
661
  try {
443
- const $ = cheerio.load(response.body);
662
+ const $ = this._cheerio.load(response.body);
444
663
  // Remove unwanted tags
445
664
  for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside']) {
446
665
  $(tag).remove();
@@ -484,7 +703,7 @@ export class SpiderSkill extends SkillBase {
484
703
  _structuredExtract(cached, selectors) {
485
704
  let $;
486
705
  try {
487
- $ = cheerio.load(cached.body);
706
+ $ = this._cheerio.load(cached.body);
488
707
  }
489
708
  catch (err) {
490
709
  // Python returns `{'error': str(e)}` from _structured_extract when the
@@ -542,13 +761,12 @@ export class SpiderSkill extends SkillBase {
542
761
  return new FunctionResult(`Invalid URL: ${url}`);
543
762
  }
544
763
  // SSRF protection
545
- const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
546
- try {
547
- await resolveAndValidateUrl(url, allowPrivate);
548
- }
549
- catch {
764
+ if (!(await SpiderSkill._targetAllowed(url))) {
550
765
  return new FunctionResult('URL rejected: cannot access private or internal URLs');
551
766
  }
767
+ if (!(await this._allowedByRobots(url))) {
768
+ return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
769
+ }
552
770
  const cached = await this._fetchUrl(url);
553
771
  if (!cached) {
554
772
  return new FunctionResult(`Failed to fetch ${url}`);
@@ -585,11 +803,7 @@ export class SpiderSkill extends SkillBase {
585
803
  if (!/^https?:\/\//.test(startUrl)) {
586
804
  return new FunctionResult(`Invalid URL: ${startUrl}`);
587
805
  }
588
- const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
589
- try {
590
- await resolveAndValidateUrl(startUrl, allowPrivate);
591
- }
592
- catch {
806
+ if (!(await SpiderSkill._targetAllowed(startUrl))) {
593
807
  return new FunctionResult('URL rejected: cannot access private or internal URLs');
594
808
  }
595
809
  const maxDepth = this.maxDepth;
@@ -599,6 +813,7 @@ export class SpiderSkill extends SkillBase {
599
813
  if (maxPages < 1)
600
814
  return new FunctionResult('Max pages must be at least 1');
601
815
  const visited = new Set();
816
+ const disallowed = new Set();
602
817
  const toVisit = [[startUrl, 0]];
603
818
  const results = [];
604
819
  let startHost;
@@ -613,8 +828,13 @@ export class SpiderSkill extends SkillBase {
613
828
  if (!next)
614
829
  break;
615
830
  const [url, depth] = next;
616
- if (visited.has(url) || depth > maxDepth)
831
+ if (visited.has(url) || disallowed.has(url) || depth > maxDepth)
617
832
  continue;
833
+ if (!(await this._allowedByRobots(url))) {
834
+ log.info('spider: robots.txt disallows a page; skipping it', { url });
835
+ disallowed.add(url);
836
+ continue;
837
+ }
618
838
  const cached = await this._fetchUrl(url);
619
839
  if (!cached)
620
840
  continue;
@@ -634,7 +854,7 @@ export class SpiderSkill extends SkillBase {
634
854
  // xpath('//a[@href]/@href') which has the same robustness).
635
855
  if (depth < maxDepth) {
636
856
  try {
637
- const $links = cheerio.load(cached.body);
857
+ const $links = this._cheerio.load(cached.body);
638
858
  const hrefs = [];
639
859
  $links('a[href]').each((_, el) => {
640
860
  const href = $links(el).attr('href');
@@ -690,10 +910,7 @@ export class SpiderSkill extends SkillBase {
690
910
  if (results.length === 0) {
691
911
  return new FunctionResult(`No pages could be crawled from ${startUrl}`);
692
912
  }
693
- const lines = [
694
- `Crawled ${results.length} pages from ${startHost}:`,
695
- '',
696
- ];
913
+ const lines = [`Crawled ${results.length} pages from ${startHost}:`, ''];
697
914
  results.forEach((r, i) => {
698
915
  lines.push(`${i + 1}. ${r.url} (depth: ${r.depth}, ${r.contentLength} chars)`);
699
916
  lines.push(` Summary: ${r.summary.slice(0, 100)}...`);
@@ -713,16 +930,15 @@ export class SpiderSkill extends SkillBase {
713
930
  if (!/^https?:\/\//.test(url)) {
714
931
  return new FunctionResult(`Invalid URL: ${url}`);
715
932
  }
716
- const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
717
- try {
718
- await resolveAndValidateUrl(url, allowPrivate);
719
- }
720
- catch {
933
+ if (!(await SpiderSkill._targetAllowed(url))) {
721
934
  return new FunctionResult('URL rejected: cannot access private or internal URLs');
722
935
  }
723
936
  if (!this.selectors || Object.keys(this.selectors).length === 0) {
724
937
  return new FunctionResult('No selectors configured for structured data extraction');
725
938
  }
939
+ if (!(await this._allowedByRobots(url))) {
940
+ return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
941
+ }
726
942
  const cached = await this._fetchUrl(url);
727
943
  if (!cached) {
728
944
  return new FunctionResult(`Failed to fetch ${url}`);