@signalwire/sdk 2.0.5 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (656) hide show
  1. package/CHANGELOG.md +785 -0
  2. package/LICENSE +21 -0
  3. package/README.md +163 -68
  4. package/dist/AGENTS.md +13 -0
  5. package/dist/AgentBase.d.ts +481 -126
  6. package/dist/AgentBase.js +1502 -438
  7. package/dist/AgentBase.js.map +1 -1
  8. package/dist/AgentServer.d.ts +21 -0
  9. package/dist/AgentServer.js +154 -47
  10. package/dist/AgentServer.js.map +1 -1
  11. package/dist/AuthHandler.d.ts +14 -2
  12. package/dist/AuthHandler.js +7 -2
  13. package/dist/AuthHandler.js.map +1 -1
  14. package/dist/ConfigLoader.d.ts +5 -13
  15. package/dist/ConfigLoader.js +15 -29
  16. package/dist/ConfigLoader.js.map +1 -1
  17. package/dist/ContextBuilder.d.ts +202 -9
  18. package/dist/ContextBuilder.js +219 -36
  19. package/dist/ContextBuilder.js.map +1 -1
  20. package/dist/DataMap.d.ts +158 -45
  21. package/dist/DataMap.js +167 -52
  22. package/dist/DataMap.js.map +1 -1
  23. package/dist/FunctionResult.d.ts +277 -83
  24. package/dist/FunctionResult.js +444 -116
  25. package/dist/FunctionResult.js.map +1 -1
  26. package/dist/Logger.d.ts +30 -5
  27. package/dist/Logger.js +77 -19
  28. package/dist/Logger.js.map +1 -1
  29. package/dist/POM/PromptObjectModel.d.ts +12 -1
  30. package/dist/POM/PromptObjectModel.js +25 -9
  31. package/dist/POM/PromptObjectModel.js.map +1 -1
  32. package/dist/ParameterSchema.d.ts +306 -0
  33. package/dist/ParameterSchema.js +251 -0
  34. package/dist/ParameterSchema.js.map +1 -0
  35. package/dist/PlatformContracts.d.ts +21 -0
  36. package/dist/PlatformContracts.generated.d.ts +109 -0
  37. package/dist/PlatformContracts.generated.js +8 -0
  38. package/dist/PlatformContracts.generated.js.map +1 -0
  39. package/dist/PlatformContracts.js +2 -0
  40. package/dist/PlatformContracts.js.map +1 -0
  41. package/dist/PomBuilder.d.ts +18 -0
  42. package/dist/PomBuilder.js +53 -3
  43. package/dist/PomBuilder.js.map +1 -1
  44. package/dist/PostPrompt.d.ts +110 -0
  45. package/dist/PostPrompt.js +204 -0
  46. package/dist/PostPrompt.js.map +1 -0
  47. package/dist/PromptManager.d.ts +22 -1
  48. package/dist/PromptManager.js +28 -1
  49. package/dist/PromptManager.js.map +1 -1
  50. package/dist/PublicFetch.d.ts +132 -0
  51. package/dist/PublicFetch.js +311 -0
  52. package/dist/PublicFetch.js.map +1 -0
  53. package/dist/RobotsTxt.d.ts +41 -0
  54. package/dist/RobotsTxt.js +152 -0
  55. package/dist/RobotsTxt.js.map +1 -0
  56. package/dist/SWMLHandler.js.map +1 -1
  57. package/dist/SWMLService.d.ts +158 -58
  58. package/dist/SWMLService.js +435 -127
  59. package/dist/SWMLService.js.map +1 -1
  60. package/dist/SchemaUtils.d.ts +115 -2
  61. package/dist/SchemaUtils.js +444 -12
  62. package/dist/SchemaUtils.js.map +1 -1
  63. package/dist/SecurityUtils.d.ts +49 -9
  64. package/dist/SecurityUtils.js +153 -51
  65. package/dist/SecurityUtils.js.map +1 -1
  66. package/dist/ServerlessAdapter.d.ts +156 -7
  67. package/dist/ServerlessAdapter.js +399 -42
  68. package/dist/ServerlessAdapter.js.map +1 -1
  69. package/dist/SessionManager.d.ts +11 -2
  70. package/dist/SessionManager.js +50 -18
  71. package/dist/SessionManager.js.map +1 -1
  72. package/dist/SslConfig.d.ts +2 -1
  73. package/dist/SslConfig.js +4 -1
  74. package/dist/SslConfig.js.map +1 -1
  75. package/dist/SwaigActions.generated.d.ts +23 -0
  76. package/dist/SwaigActions.generated.js +8 -0
  77. package/dist/SwaigActions.generated.js.map +1 -0
  78. package/dist/SwaigContracts.d.ts +15 -0
  79. package/dist/SwaigContracts.generated.d.ts +264 -0
  80. package/dist/SwaigContracts.generated.js +12 -0
  81. package/dist/SwaigContracts.generated.js.map +1 -0
  82. package/dist/SwaigContracts.js +2 -0
  83. package/dist/SwaigContracts.js.map +1 -0
  84. package/dist/SwaigFunction.d.ts +110 -4
  85. package/dist/SwaigFunction.js +151 -19
  86. package/dist/SwaigFunction.js.map +1 -1
  87. package/dist/SwmlBuilder.d.ts +68 -19
  88. package/dist/SwmlBuilder.js +74 -38
  89. package/dist/SwmlBuilder.js.map +1 -1
  90. package/dist/SwmlVerbMethods.generated.d.ts +41 -4
  91. package/dist/SwmlVerbMethods.generated.js +1 -1
  92. package/dist/TypeInference.d.ts +39 -3
  93. package/dist/TypeInference.js +256 -16
  94. package/dist/TypeInference.js.map +1 -1
  95. package/dist/WebService.d.ts +79 -11
  96. package/dist/WebService.js +320 -109
  97. package/dist/WebService.js.map +1 -1
  98. package/dist/WebhookMiddleware.d.ts +119 -0
  99. package/dist/WebhookMiddleware.js +271 -0
  100. package/dist/WebhookMiddleware.js.map +1 -0
  101. package/dist/WebhookValidator.d.ts +92 -0
  102. package/dist/WebhookValidator.js +409 -0
  103. package/dist/WebhookValidator.js.map +1 -0
  104. package/dist/agents/BedrockAgent.d.ts +162 -0
  105. package/dist/agents/BedrockAgent.js +323 -0
  106. package/dist/agents/BedrockAgent.js.map +1 -0
  107. package/dist/ai-chat/AIChatClient.d.ts +316 -0
  108. package/dist/ai-chat/AIChatClient.js +535 -0
  109. package/dist/ai-chat/AIChatClient.js.map +1 -0
  110. package/dist/ai-chat/ChatGateway.d.ts +408 -0
  111. package/dist/ai-chat/ChatGateway.js +747 -0
  112. package/dist/ai-chat/ChatGateway.js.map +1 -0
  113. package/dist/ai-chat/HandoffRouter.d.ts +282 -0
  114. package/dist/ai-chat/HandoffRouter.js +518 -0
  115. package/dist/ai-chat/HandoffRouter.js.map +1 -0
  116. package/dist/ai-chat/index.d.ts +12 -0
  117. package/dist/ai-chat/index.js +10 -0
  118. package/dist/ai-chat/index.js.map +1 -0
  119. package/dist/capabilities.d.ts +64 -0
  120. package/dist/capabilities.js +95 -0
  121. package/dist/capabilities.js.map +1 -0
  122. package/dist/cli/agent-loader.d.ts +32 -6
  123. package/dist/cli/agent-loader.js +92 -33
  124. package/dist/cli/agent-loader.js.map +1 -1
  125. package/dist/cli/datamap-exec.d.ts +175 -0
  126. package/dist/cli/datamap-exec.js +1284 -0
  127. package/dist/cli/datamap-exec.js.map +1 -0
  128. package/dist/cli/function-args.d.ts +40 -0
  129. package/dist/cli/function-args.js +146 -0
  130. package/dist/cli/function-args.js.map +1 -0
  131. package/dist/cli/simulation.d.ts +99 -0
  132. package/dist/cli/simulation.js +490 -0
  133. package/dist/cli/simulation.js.map +1 -0
  134. package/dist/cli/swaig-test.d.ts +12 -25
  135. package/dist/cli/swaig-test.js +935 -371
  136. package/dist/cli/swaig-test.js.map +1 -1
  137. package/dist/cli/tsdocs/agentsNote.d.ts +30 -0
  138. package/dist/cli/tsdocs/agentsNote.js +92 -0
  139. package/dist/cli/tsdocs/agentsNote.js.map +1 -0
  140. package/dist/cli/tsdocs/api-index.json +1 -0
  141. package/dist/cli/tsdocs/api.d.ts +91 -0
  142. package/dist/cli/tsdocs/api.js +372 -0
  143. package/dist/cli/tsdocs/api.js.map +1 -0
  144. package/dist/cli/tsdocs/apiTypes.d.ts +64 -0
  145. package/dist/cli/tsdocs/apiTypes.js +7 -0
  146. package/dist/cli/tsdocs/apiTypes.js.map +1 -0
  147. package/dist/cli/tsdocs/bin.d.ts +6 -0
  148. package/dist/cli/tsdocs/bin.js +23 -0
  149. package/dist/cli/tsdocs/bin.js.map +1 -0
  150. package/dist/cli/tsdocs/files.d.ts +108 -0
  151. package/dist/cli/tsdocs/files.js +287 -0
  152. package/dist/cli/tsdocs/files.js.map +1 -0
  153. package/dist/cli/tsdocs/index.d.ts +28 -0
  154. package/dist/cli/tsdocs/index.js +307 -0
  155. package/dist/cli/tsdocs/index.js.map +1 -0
  156. package/dist/cli/tsdocs/live.d.ts +41 -0
  157. package/dist/cli/tsdocs/live.js +202 -0
  158. package/dist/cli/tsdocs/live.js.map +1 -0
  159. package/dist/cli/tsdocs/locate.d.ts +51 -0
  160. package/dist/cli/tsdocs/locate.js +75 -0
  161. package/dist/cli/tsdocs/locate.js.map +1 -0
  162. package/dist/cli/tsdocs/render.d.ts +75 -0
  163. package/dist/cli/tsdocs/render.js +323 -0
  164. package/dist/cli/tsdocs/render.js.map +1 -0
  165. package/dist/cli/tsdocs/topics.d.ts +43 -0
  166. package/dist/cli/tsdocs/topics.js +916 -0
  167. package/dist/cli/tsdocs/topics.js.map +1 -0
  168. package/dist/index.d.ts +47 -68
  169. package/dist/index.js +44 -73
  170. package/dist/index.js.map +1 -1
  171. package/dist/livewire/index.d.ts +229 -155
  172. package/dist/livewire/index.js +298 -195
  173. package/dist/livewire/index.js.map +1 -1
  174. package/dist/llms.txt +65 -0
  175. package/dist/prefabs/CallSessionStore.d.ts +52 -0
  176. package/dist/prefabs/CallSessionStore.js +72 -0
  177. package/dist/prefabs/CallSessionStore.js.map +1 -0
  178. package/dist/prefabs/ConciergeAgent.d.ts +13 -1
  179. package/dist/prefabs/ConciergeAgent.js +36 -31
  180. package/dist/prefabs/ConciergeAgent.js.map +1 -1
  181. package/dist/prefabs/FAQBotAgent.d.ts +9 -1
  182. package/dist/prefabs/FAQBotAgent.js +113 -47
  183. package/dist/prefabs/FAQBotAgent.js.map +1 -1
  184. package/dist/prefabs/InfoGathererAgent.d.ts +22 -3
  185. package/dist/prefabs/InfoGathererAgent.js +67 -58
  186. package/dist/prefabs/InfoGathererAgent.js.map +1 -1
  187. package/dist/prefabs/ReceptionistAgent.d.ts +16 -5
  188. package/dist/prefabs/ReceptionistAgent.js +39 -31
  189. package/dist/prefabs/ReceptionistAgent.js.map +1 -1
  190. package/dist/prefabs/SurveyAgent.d.ts +22 -4
  191. package/dist/prefabs/SurveyAgent.js +84 -68
  192. package/dist/prefabs/SurveyAgent.js.map +1 -1
  193. package/dist/relay/Action.d.ts +44 -21
  194. package/dist/relay/Action.js +75 -17
  195. package/dist/relay/Action.js.map +1 -1
  196. package/dist/relay/Call.d.ts +312 -47
  197. package/dist/relay/Call.js +369 -18
  198. package/dist/relay/Call.js.map +1 -1
  199. package/dist/relay/Deferred.js.map +1 -1
  200. package/dist/relay/Message.d.ts +26 -5
  201. package/dist/relay/Message.js +36 -6
  202. package/dist/relay/Message.js.map +1 -1
  203. package/dist/relay/RelayClient.d.ts +44 -17
  204. package/dist/relay/RelayClient.js +175 -65
  205. package/dist/relay/RelayClient.js.map +1 -1
  206. package/dist/relay/RelayError.d.ts +14 -0
  207. package/dist/relay/RelayError.js +15 -0
  208. package/dist/relay/RelayError.js.map +1 -1
  209. package/dist/relay/RelayEvent.d.ts +96 -65
  210. package/dist/relay/RelayEvent.js +85 -56
  211. package/dist/relay/RelayEvent.js.map +1 -1
  212. package/dist/relay/closedSets.d.ts +111 -0
  213. package/dist/relay/closedSets.js +83 -0
  214. package/dist/relay/closedSets.js.map +1 -0
  215. package/dist/relay/index.d.ts +3 -1
  216. package/dist/relay/index.js +1 -0
  217. package/dist/relay/index.js.map +1 -1
  218. package/dist/relay/normalize.js.map +1 -1
  219. package/dist/relay/protocol.types.generated.d.ts +1195 -0
  220. package/dist/relay/protocol.types.generated.js +8 -0
  221. package/dist/relay/protocol.types.generated.js.map +1 -0
  222. package/dist/relay/types.d.ts +73 -42
  223. package/dist/rest/HttpClient.d.ts +36 -8
  224. package/dist/rest/HttpClient.js +233 -40
  225. package/dist/rest/HttpClient.js.map +1 -1
  226. package/dist/rest/RequestOptions.d.ts +110 -0
  227. package/dist/rest/RequestOptions.js +113 -0
  228. package/dist/rest/RequestOptions.js.map +1 -0
  229. package/dist/rest/RestError.d.ts +52 -6
  230. package/dist/rest/RestError.js +89 -5
  231. package/dist/rest/RestError.js.map +1 -1
  232. package/dist/rest/base/CrudResource.d.ts +16 -26
  233. package/dist/rest/base/CrudResource.js +19 -33
  234. package/dist/rest/base/CrudResource.js.map +1 -1
  235. package/dist/rest/base/CrudWithAddresses.d.ts +3 -2
  236. package/dist/rest/base/CrudWithAddresses.js +2 -2
  237. package/dist/rest/base/CrudWithAddresses.js.map +1 -1
  238. package/dist/rest/base/FabricResource.d.ts +15 -0
  239. package/dist/rest/base/FabricResource.js +16 -0
  240. package/dist/rest/base/FabricResource.js.map +1 -0
  241. package/dist/rest/base/ReadResource.d.ts +62 -0
  242. package/dist/rest/base/ReadResource.js +69 -0
  243. package/dist/rest/base/ReadResource.js.map +1 -0
  244. package/dist/rest/callHandler.d.ts +5 -2
  245. package/dist/rest/callHandler.js +4 -1
  246. package/dist/rest/callHandler.js.map +1 -1
  247. package/dist/rest/index.d.ts +18 -77
  248. package/dist/rest/index.js +47 -120
  249. package/dist/rest/index.js.map +1 -1
  250. package/dist/rest/namespaces/_client_tree_generated.d.ts +103 -0
  251. package/dist/rest/namespaces/_client_tree_generated.js +171 -0
  252. package/dist/rest/namespaces/_client_tree_generated.js.map +1 -0
  253. package/dist/rest/namespaces/addresses.d.ts +3 -42
  254. package/dist/rest/namespaces/addresses.js +3 -50
  255. package/dist/rest/namespaces/addresses.js.map +1 -1
  256. package/dist/rest/namespaces/calling.d.ts +6 -378
  257. package/dist/rest/namespaces/calling.js +6 -473
  258. package/dist/rest/namespaces/calling.js.map +1 -1
  259. package/dist/rest/namespaces/calling.resources.generated.d.ts +165 -0
  260. package/dist/rest/namespaces/calling.resources.generated.js +477 -0
  261. package/dist/rest/namespaces/calling.resources.generated.js.map +1 -0
  262. package/dist/rest/namespaces/calling.types.generated.d.ts +2917 -0
  263. package/dist/rest/namespaces/calling.types.generated.js +8 -0
  264. package/dist/rest/namespaces/calling.types.generated.js.map +1 -0
  265. package/dist/rest/namespaces/chat.d.ts +4 -18
  266. package/dist/rest/namespaces/chat.js +4 -21
  267. package/dist/rest/namespaces/chat.js.map +1 -1
  268. package/dist/rest/namespaces/chat.resources.generated.d.ts +12 -0
  269. package/dist/rest/namespaces/chat.resources.generated.js +29 -0
  270. package/dist/rest/namespaces/chat.resources.generated.js.map +1 -0
  271. package/dist/rest/namespaces/chat.types.generated.d.ts +56 -0
  272. package/dist/rest/namespaces/chat.types.generated.js +8 -0
  273. package/dist/rest/namespaces/chat.types.generated.js.map +1 -0
  274. package/dist/rest/namespaces/datasphere.d.ts +6 -62
  275. package/dist/rest/namespaces/datasphere.js +6 -72
  276. package/dist/rest/namespaces/datasphere.js.map +1 -1
  277. package/dist/rest/namespaces/datasphere.resources.generated.d.ts +25 -0
  278. package/dist/rest/namespaces/datasphere.resources.generated.js +50 -0
  279. package/dist/rest/namespaces/datasphere.resources.generated.js.map +1 -0
  280. package/dist/rest/namespaces/datasphere.types.generated.d.ts +180 -0
  281. package/dist/rest/namespaces/datasphere.types.generated.js +8 -0
  282. package/dist/rest/namespaces/datasphere.types.generated.js.map +1 -0
  283. package/dist/rest/namespaces/fabric.d.ts +11 -327
  284. package/dist/rest/namespaces/fabric.js +14 -439
  285. package/dist/rest/namespaces/fabric.js.map +1 -1
  286. package/dist/rest/namespaces/fabric.resources.generated.d.ts +195 -0
  287. package/dist/rest/namespaces/fabric.resources.generated.js +399 -0
  288. package/dist/rest/namespaces/fabric.resources.generated.js.map +1 -0
  289. package/dist/rest/namespaces/fabric.types.generated.d.ts +4557 -0
  290. package/dist/rest/namespaces/fabric.types.generated.js +8 -0
  291. package/dist/rest/namespaces/fabric.types.generated.js.map +1 -0
  292. package/dist/rest/namespaces/fax.resources.generated.d.ts +6 -0
  293. package/dist/rest/namespaces/fax.resources.generated.js +14 -0
  294. package/dist/rest/namespaces/fax.resources.generated.js.map +1 -0
  295. package/dist/rest/namespaces/fax.types.generated.d.ts +131 -0
  296. package/dist/rest/namespaces/fax.types.generated.js +8 -0
  297. package/dist/rest/namespaces/fax.types.generated.js.map +1 -0
  298. package/dist/rest/namespaces/imported-numbers.d.ts +3 -18
  299. package/dist/rest/namespaces/imported-numbers.js +3 -21
  300. package/dist/rest/namespaces/imported-numbers.js.map +1 -1
  301. package/dist/rest/namespaces/logs.d.ts +10 -101
  302. package/dist/rest/namespaces/logs.js +10 -128
  303. package/dist/rest/namespaces/logs.js.map +1 -1
  304. package/dist/rest/namespaces/logs.resources.generated.d.ts +9 -0
  305. package/dist/rest/namespaces/logs.resources.generated.js +17 -0
  306. package/dist/rest/namespaces/logs.resources.generated.js.map +1 -0
  307. package/dist/rest/namespaces/logs.types.generated.d.ts +131 -0
  308. package/dist/rest/namespaces/logs.types.generated.js +8 -0
  309. package/dist/rest/namespaces/logs.types.generated.js.map +1 -0
  310. package/dist/rest/namespaces/lookup.d.ts +3 -25
  311. package/dist/rest/namespaces/lookup.js +3 -27
  312. package/dist/rest/namespaces/lookup.js.map +1 -1
  313. package/dist/rest/namespaces/message.resources.generated.d.ts +6 -0
  314. package/dist/rest/namespaces/message.resources.generated.js +14 -0
  315. package/dist/rest/namespaces/message.resources.generated.js.map +1 -0
  316. package/dist/rest/namespaces/message.types.generated.d.ts +122 -0
  317. package/dist/rest/namespaces/message.types.generated.js +8 -0
  318. package/dist/rest/namespaces/message.types.generated.js.map +1 -0
  319. package/dist/rest/namespaces/messages.resources.generated.d.ts +18 -0
  320. package/dist/rest/namespaces/messages.resources.generated.js +44 -0
  321. package/dist/rest/namespaces/messages.resources.generated.js.map +1 -0
  322. package/dist/rest/namespaces/messages.types.generated.d.ts +99 -0
  323. package/dist/rest/namespaces/messages.types.generated.js +8 -0
  324. package/dist/rest/namespaces/messages.types.generated.js.map +1 -0
  325. package/dist/rest/namespaces/mfa.d.ts +3 -42
  326. package/dist/rest/namespaces/mfa.js +3 -49
  327. package/dist/rest/namespaces/mfa.js.map +1 -1
  328. package/dist/rest/namespaces/number-groups.d.ts +3 -45
  329. package/dist/rest/namespaces/number-groups.js +3 -53
  330. package/dist/rest/namespaces/number-groups.js.map +1 -1
  331. package/dist/rest/namespaces/phone-numbers.d.ts +8 -196
  332. package/dist/rest/namespaces/phone-numbers.js +8 -230
  333. package/dist/rest/namespaces/phone-numbers.js.map +1 -1
  334. package/dist/rest/namespaces/project.d.ts +6 -42
  335. package/dist/rest/namespaces/project.js +6 -51
  336. package/dist/rest/namespaces/project.js.map +1 -1
  337. package/dist/rest/namespaces/project.resources.generated.d.ts +17 -0
  338. package/dist/rest/namespaces/project.resources.generated.js +44 -0
  339. package/dist/rest/namespaces/project.resources.generated.js.map +1 -0
  340. package/dist/rest/namespaces/project.types.generated.d.ts +58 -0
  341. package/dist/rest/namespaces/project.types.generated.js +8 -0
  342. package/dist/rest/namespaces/project.types.generated.js.map +1 -0
  343. package/dist/rest/namespaces/projects.resources.generated.d.ts +12 -0
  344. package/dist/rest/namespaces/projects.resources.generated.js +25 -0
  345. package/dist/rest/namespaces/projects.resources.generated.js.map +1 -0
  346. package/dist/rest/namespaces/projects.types.generated.d.ts +77 -0
  347. package/dist/rest/namespaces/projects.types.generated.js +8 -0
  348. package/dist/rest/namespaces/projects.types.generated.js.map +1 -0
  349. package/dist/rest/namespaces/pubsub.d.ts +4 -18
  350. package/dist/rest/namespaces/pubsub.js +4 -21
  351. package/dist/rest/namespaces/pubsub.js.map +1 -1
  352. package/dist/rest/namespaces/pubsub.resources.generated.d.ts +12 -0
  353. package/dist/rest/namespaces/pubsub.resources.generated.js +29 -0
  354. package/dist/rest/namespaces/pubsub.resources.generated.js.map +1 -0
  355. package/dist/rest/namespaces/pubsub.types.generated.d.ts +60 -0
  356. package/dist/rest/namespaces/pubsub.types.generated.js +8 -0
  357. package/dist/rest/namespaces/pubsub.types.generated.js.map +1 -0
  358. package/dist/rest/namespaces/queues.d.ts +3 -38
  359. package/dist/rest/namespaces/queues.js +3 -44
  360. package/dist/rest/namespaces/queues.js.map +1 -1
  361. package/dist/rest/namespaces/recordings.d.ts +3 -34
  362. package/dist/rest/namespaces/recordings.js +3 -40
  363. package/dist/rest/namespaces/recordings.js.map +1 -1
  364. package/dist/rest/namespaces/registry.d.ts +6 -137
  365. package/dist/rest/namespaces/registry.js +6 -173
  366. package/dist/rest/namespaces/registry.js.map +1 -1
  367. package/dist/rest/namespaces/relay-rest.resources.generated.d.ts +168 -0
  368. package/dist/rest/namespaces/relay-rest.resources.generated.js +430 -0
  369. package/dist/rest/namespaces/relay-rest.resources.generated.js.map +1 -0
  370. package/dist/rest/namespaces/relay-rest.types.generated.d.ts +1906 -0
  371. package/dist/rest/namespaces/relay-rest.types.generated.js +8 -0
  372. package/dist/rest/namespaces/relay-rest.types.generated.js.map +1 -0
  373. package/dist/rest/namespaces/short-codes.d.ts +3 -35
  374. package/dist/rest/namespaces/short-codes.js +3 -41
  375. package/dist/rest/namespaces/short-codes.js.map +1 -1
  376. package/dist/rest/namespaces/sip-profile.d.ts +3 -24
  377. package/dist/rest/namespaces/sip-profile.js +3 -29
  378. package/dist/rest/namespaces/sip-profile.js.map +1 -1
  379. package/dist/rest/namespaces/verified-callers.d.ts +3 -29
  380. package/dist/rest/namespaces/verified-callers.js +3 -34
  381. package/dist/rest/namespaces/verified-callers.js.map +1 -1
  382. package/dist/rest/namespaces/video.d.ts +10 -227
  383. package/dist/rest/namespaces/video.js +10 -288
  384. package/dist/rest/namespaces/video.js.map +1 -1
  385. package/dist/rest/namespaces/video.resources.generated.d.ts +81 -0
  386. package/dist/rest/namespaces/video.resources.generated.js +171 -0
  387. package/dist/rest/namespaces/video.resources.generated.js.map +1 -0
  388. package/dist/rest/namespaces/video.types.generated.d.ts +771 -0
  389. package/dist/rest/namespaces/video.types.generated.js +8 -0
  390. package/dist/rest/namespaces/video.types.generated.js.map +1 -0
  391. package/dist/rest/namespaces/voice.resources.generated.d.ts +9 -0
  392. package/dist/rest/namespaces/voice.resources.generated.js +17 -0
  393. package/dist/rest/namespaces/voice.resources.generated.js.map +1 -0
  394. package/dist/rest/namespaces/voice.types.generated.d.ts +214 -0
  395. package/dist/rest/namespaces/voice.types.generated.js +8 -0
  396. package/dist/rest/namespaces/voice.types.generated.js.map +1 -0
  397. package/dist/rest/pagination.d.ts +5 -1
  398. package/dist/rest/pagination.js +40 -20
  399. package/dist/rest/pagination.js.map +1 -1
  400. package/dist/rest/types.d.ts +21 -3
  401. package/dist/schema.json +299 -98
  402. package/dist/skills/SkillBase.d.ts +97 -20
  403. package/dist/skills/SkillBase.js +100 -17
  404. package/dist/skills/SkillBase.js.map +1 -1
  405. package/dist/skills/SkillManager.d.ts +66 -13
  406. package/dist/skills/SkillManager.js +101 -20
  407. package/dist/skills/SkillManager.js.map +1 -1
  408. package/dist/skills/SkillName.d.ts +44 -0
  409. package/dist/skills/SkillName.js +2 -0
  410. package/dist/skills/SkillName.js.map +1 -0
  411. package/dist/skills/SkillRegistry.d.ts +22 -11
  412. package/dist/skills/SkillRegistry.js +47 -17
  413. package/dist/skills/SkillRegistry.js.map +1 -1
  414. package/dist/skills/builtin/api_ninjas_trivia.d.ts +3 -1
  415. package/dist/skills/builtin/api_ninjas_trivia.js +18 -6
  416. package/dist/skills/builtin/api_ninjas_trivia.js.map +1 -1
  417. package/dist/skills/builtin/ask_claude.d.ts +14 -4
  418. package/dist/skills/builtin/ask_claude.js +36 -14
  419. package/dist/skills/builtin/ask_claude.js.map +1 -1
  420. package/dist/skills/builtin/claude_skills.d.ts +3 -1
  421. package/dist/skills/builtin/claude_skills.js +19 -15
  422. package/dist/skills/builtin/claude_skills.js.map +1 -1
  423. package/dist/skills/builtin/custom_skills.d.ts +10 -3
  424. package/dist/skills/builtin/custom_skills.js +26 -9
  425. package/dist/skills/builtin/custom_skills.js.map +1 -1
  426. package/dist/skills/builtin/datasphere.d.ts +5 -3
  427. package/dist/skills/builtin/datasphere.js +21 -22
  428. package/dist/skills/builtin/datasphere.js.map +1 -1
  429. package/dist/skills/builtin/datasphere_serverless.d.ts +12 -1
  430. package/dist/skills/builtin/datasphere_serverless.js +17 -6
  431. package/dist/skills/builtin/datasphere_serverless.js.map +1 -1
  432. package/dist/skills/builtin/datetime.d.ts +3 -1
  433. package/dist/skills/builtin/datetime.js +11 -7
  434. package/dist/skills/builtin/datetime.js.map +1 -1
  435. package/dist/skills/builtin/google_maps.d.ts +29 -14
  436. package/dist/skills/builtin/google_maps.js +89 -269
  437. package/dist/skills/builtin/google_maps.js.map +1 -1
  438. package/dist/skills/builtin/index.d.ts +8 -6
  439. package/dist/skills/builtin/index.js +8 -6
  440. package/dist/skills/builtin/index.js.map +1 -1
  441. package/dist/skills/builtin/info_gatherer.d.ts +4 -2
  442. package/dist/skills/builtin/info_gatherer.js +20 -12
  443. package/dist/skills/builtin/info_gatherer.js.map +1 -1
  444. package/dist/skills/builtin/joke.d.ts +16 -6
  445. package/dist/skills/builtin/joke.js +54 -37
  446. package/dist/skills/builtin/joke.js.map +1 -1
  447. package/dist/skills/builtin/math.d.ts +3 -1
  448. package/dist/skills/builtin/math.js +14 -9
  449. package/dist/skills/builtin/math.js.map +1 -1
  450. package/dist/skills/builtin/mcp_gateway.d.ts +34 -5
  451. package/dist/skills/builtin/mcp_gateway.js +77 -46
  452. package/dist/skills/builtin/mcp_gateway.js.map +1 -1
  453. package/dist/skills/builtin/native_vector_search.d.ts +19 -5
  454. package/dist/skills/builtin/native_vector_search.js +179 -61
  455. package/dist/skills/builtin/native_vector_search.js.map +1 -1
  456. package/dist/skills/builtin/play_background_file.d.ts +21 -15
  457. package/dist/skills/builtin/play_background_file.js +43 -122
  458. package/dist/skills/builtin/play_background_file.js.map +1 -1
  459. package/dist/skills/builtin/spider.d.ts +42 -4
  460. package/dist/skills/builtin/spider.js +277 -104
  461. package/dist/skills/builtin/spider.js.map +1 -1
  462. package/dist/skills/builtin/swml_transfer.d.ts +14 -3
  463. package/dist/skills/builtin/swml_transfer.js +45 -24
  464. package/dist/skills/builtin/swml_transfer.js.map +1 -1
  465. package/dist/skills/builtin/weather_api.d.ts +24 -13
  466. package/dist/skills/builtin/weather_api.js +38 -30
  467. package/dist/skills/builtin/weather_api.js.map +1 -1
  468. package/dist/skills/builtin/web_search.d.ts +48 -10
  469. package/dist/skills/builtin/web_search.js +294 -100
  470. package/dist/skills/builtin/web_search.js.map +1 -1
  471. package/dist/skills/builtin/wikipedia_search.d.ts +4 -2
  472. package/dist/skills/builtin/wikipedia_search.js +14 -13
  473. package/dist/skills/builtin/wikipedia_search.js.map +1 -1
  474. package/dist/skills/index.d.ts +3 -2
  475. package/dist/skills/index.js +1 -1
  476. package/dist/skills/index.js.map +1 -1
  477. package/dist/swml_verbs_generated.d.ts +2824 -0
  478. package/dist/swml_verbs_generated.js +12 -0
  479. package/dist/swml_verbs_generated.js.map +1 -0
  480. package/dist/types.d.ts +59 -6
  481. package/dist/web.d.ts +27 -0
  482. package/dist/web.js +20 -0
  483. package/dist/web.js.map +1 -0
  484. package/docs/agent-guide.md +1034 -0
  485. package/docs/ai_chat.md +560 -0
  486. package/docs/api-reference.md +3534 -0
  487. package/docs/architecture.md +846 -0
  488. package/docs/bedrock_agent.md +411 -0
  489. package/docs/cli-guide.md +403 -0
  490. package/docs/cloud_functions_guide.md +399 -0
  491. package/docs/configuration.md +548 -0
  492. package/docs/contexts-guide.md +884 -0
  493. package/docs/datamap-guide.md +1160 -0
  494. package/docs/developer_pain_points.md +548 -0
  495. package/docs/llm_parameters.md +298 -0
  496. package/docs/mcp_gateway_reference.md +354 -0
  497. package/docs/mcp_integration.md +162 -0
  498. package/docs/pgi_agent_guide.md +1820 -0
  499. package/docs/prefabs-guide.md +815 -0
  500. package/docs/programmatically_governed_inference.md +301 -0
  501. package/docs/sdk_features.md +501 -0
  502. package/docs/security.md +559 -0
  503. package/docs/serverless-guide.md +520 -0
  504. package/docs/skills-guide.md +1223 -0
  505. package/docs/skills-system.md +405 -0
  506. package/docs/skills_parameter_schema.md +417 -0
  507. package/docs/swaig-reference.md +1938 -0
  508. package/docs/swml_service_guide.md +573 -0
  509. package/docs/third_party_skills.md +410 -0
  510. package/docs/web_service.md +666 -0
  511. package/examples/README.md +202 -0
  512. package/examples/advanced-datamap.ts +108 -0
  513. package/examples/advanced-dynamic-config.ts +70 -0
  514. package/examples/auto-vivified.ts +95 -0
  515. package/examples/call-flow.ts +65 -0
  516. package/examples/comprehensive-dynamic.ts +125 -0
  517. package/examples/contexts-steps.ts +65 -0
  518. package/examples/custom-path.ts +45 -0
  519. package/examples/datamap-tools.ts +51 -0
  520. package/examples/datasphere-multi-instance.ts +66 -0
  521. package/examples/datasphere-serverless-env.ts +87 -0
  522. package/examples/datasphere-webhook-env.ts +65 -0
  523. package/examples/datasphere.ts +56 -0
  524. package/examples/declarative.ts +100 -0
  525. package/examples/dynamic-config.ts +54 -0
  526. package/examples/dynamic-info-gatherer.ts +69 -0
  527. package/examples/dynamic-swml-service.ts +67 -0
  528. package/examples/gather-info.ts +95 -0
  529. package/examples/gather-per-question-functions-demo.ts +139 -0
  530. package/examples/joke-agent.ts +34 -0
  531. package/examples/kubernetes-agent.ts +45 -0
  532. package/examples/llm-params.ts +75 -0
  533. package/examples/mcp-agent.ts +89 -0
  534. package/examples/mcp-gateway.ts +81 -0
  535. package/examples/multi-agent.ts +80 -0
  536. package/examples/multi-endpoint.ts +79 -0
  537. package/examples/pgi/case-domain.ts +114 -0
  538. package/examples/pgi/serve.ts +16 -0
  539. package/examples/pgi/support-agent.ts +265 -0
  540. package/examples/pom-prompt.ts +70 -0
  541. package/examples/prefab-concierge.ts +42 -0
  542. package/examples/prefab-faq.ts +58 -0
  543. package/examples/prefab-info-gatherer.ts +33 -0
  544. package/examples/prefab-receptionist.ts +36 -0
  545. package/examples/prefab-survey.ts +69 -0
  546. package/examples/quickstart-agent.ts +29 -0
  547. package/examples/quickstart-relay.ts +25 -0
  548. package/examples/quickstart-rest.ts +25 -0
  549. package/examples/record-call.ts +65 -0
  550. package/examples/relay-demo.ts +33 -0
  551. package/examples/room-and-sip.ts +93 -0
  552. package/examples/serverless-lambda.ts +43 -0
  553. package/examples/session-state.ts +83 -0
  554. package/examples/simple-agent.ts +36 -0
  555. package/examples/simple-static.ts +29 -0
  556. package/examples/skills-demo.ts +41 -0
  557. package/examples/step-function-inheritance-demo.ts +124 -0
  558. package/examples/swaig-features.ts +193 -0
  559. package/examples/swml-service-routing.ts +66 -0
  560. package/examples/swml-service.ts +38 -0
  561. package/examples/swmlservice_ai_sidecar.ts +104 -0
  562. package/examples/swmlservice_swaig_standalone.ts +79 -0
  563. package/examples/tap.ts +67 -0
  564. package/examples/typed-tools.ts +68 -0
  565. package/examples/verb-methods.ts +45 -0
  566. package/examples/web-search-multi-instance.ts +75 -0
  567. package/examples/web-search.ts +38 -0
  568. package/examples/wikipedia.ts +36 -0
  569. package/livewire/README.md +164 -0
  570. package/livewire/docs/migration-guide.md +251 -0
  571. package/livewire/examples/livewire-basic-agent.ts +74 -0
  572. package/livewire/examples/livewire-handoff.ts +196 -0
  573. package/livewire/examples/livewire-multi-tool.ts +121 -0
  574. package/package.json +85 -13
  575. package/relay/README.md +101 -0
  576. package/relay/docs/call-methods.md +622 -0
  577. package/relay/docs/client-reference.md +281 -0
  578. package/relay/docs/events.md +193 -0
  579. package/relay/docs/getting-started.md +221 -0
  580. package/relay/docs/guide.md +386 -0
  581. package/relay/docs/messaging.md +246 -0
  582. package/relay/examples/relay-inbound.ts +66 -0
  583. package/relay/examples/relay-messaging.ts +74 -0
  584. package/relay/examples/relay-outbound.ts +76 -0
  585. package/rest/README.md +141 -0
  586. package/rest/docs/calling.md +313 -0
  587. package/rest/docs/client-reference.md +149 -0
  588. package/rest/docs/fabric.md +192 -0
  589. package/rest/docs/getting-started.md +138 -0
  590. package/rest/docs/guide.md +421 -0
  591. package/rest/docs/namespaces.md +367 -0
  592. package/rest/docs/phone-binding.md +116 -0
  593. package/rest/examples/rest-10dlc-registration.ts +177 -0
  594. package/rest/examples/rest-bind-phone-to-swml-webhook.ts +66 -0
  595. package/rest/examples/rest-calling-ivr-and-ai.ts +136 -0
  596. package/rest/examples/rest-calling-play-and-record.ts +98 -0
  597. package/rest/examples/rest-client.ts +59 -0
  598. package/rest/examples/rest-datasphere-search.ts +81 -0
  599. package/rest/examples/rest-fabric-conferences-and-routing.ts +147 -0
  600. package/rest/examples/rest-fabric-subscribers-and-sip.ts +106 -0
  601. package/rest/examples/rest-fabric-swml-and-callflows.ts +101 -0
  602. package/rest/examples/rest-manage-resources.ts +64 -0
  603. package/rest/examples/rest-phone-number-management.ts +200 -0
  604. package/rest/examples/rest-queues-mfa-and-recordings.ts +143 -0
  605. package/rest/examples/rest-video-rooms.ts +186 -0
  606. package/tutorial/fred/fred.ts +218 -0
  607. package/tutorial/fred/tutorial/01-introduction.md +141 -0
  608. package/tutorial/fred/tutorial/02-setup.md +304 -0
  609. package/tutorial/fred/tutorial/03-basic-agent.md +284 -0
  610. package/tutorial/fred/tutorial/04-wikipedia-skill.md +298 -0
  611. package/tutorial/fred/tutorial/05-custom-functions.md +347 -0
  612. package/tutorial/fred/tutorial/06-running-testing.md +642 -0
  613. package/tutorial/fred/tutorial/README.md +56 -0
  614. package/tutorial/fred/tutorial/appendix-complete-code.md +632 -0
  615. package/tutorial/fred/tutorial/appendix-docker-deployment.md +504 -0
  616. package/tutorial/full-guardrails-agent/handlers.ts +436 -0
  617. package/tutorial/full-guardrails-agent/penny.ts +305 -0
  618. package/tutorial/full-guardrails-agent/reservations.ts +1109 -0
  619. package/tutorial/full-guardrails-agent/tutorial/01-why-guardrails.md +149 -0
  620. package/tutorial/full-guardrails-agent/tutorial/02-design-first.md +139 -0
  621. package/tutorial/full-guardrails-agent/tutorial/03-rules-first.md +422 -0
  622. package/tutorial/full-guardrails-agent/tutorial/04-the-shell.md +238 -0
  623. package/tutorial/full-guardrails-agent/tutorial/05-steps-and-scoping.md +247 -0
  624. package/tutorial/full-guardrails-agent/tutorial/06-tools-that-decide.md +467 -0
  625. package/tutorial/full-guardrails-agent/tutorial/07-gather-and-projection.md +160 -0
  626. package/tutorial/full-guardrails-agent/tutorial/08-the-verification-gate.md +333 -0
  627. package/tutorial/full-guardrails-agent/tutorial/09-people-and-endings.md +313 -0
  628. package/tutorial/full-guardrails-agent/tutorial/10-testing-and-running.md +387 -0
  629. package/tutorial/full-guardrails-agent/tutorial/README.md +106 -0
  630. package/tutorial/full-guardrails-agent/tutorial/appendix-complete-code.md +2207 -0
  631. package/tutorial/full-guardrails-agent/tutorial/appendix-deployment.md +191 -0
  632. package/tutorial/full-guardrails-agent/tutorial/appendix-technique-map.md +117 -0
  633. package/tutorial/full-guardrails-agent/workflow.ts +219 -0
  634. package/tutorial/multi_agents/README.md +175 -0
  635. package/tutorial/multi_agents/advanced_agent.ts +395 -0
  636. package/tutorial/multi_agents/extending_agents.ts +641 -0
  637. package/tutorial/multi_agents/knowledge.ts +59 -0
  638. package/tutorial/multi_agents/lesson1_first_agent.md +368 -0
  639. package/tutorial/multi_agents/lesson2_knowledge_bases.md +404 -0
  640. package/tutorial/multi_agents/lesson3_multi_agent_systems.md +621 -0
  641. package/tutorial/multi_agents/lesson4_advanced_features.md +805 -0
  642. package/tutorial/multi_agents/lesson5_extending_agents.md +856 -0
  643. package/tutorial/multi_agents/pc_builder.ts +529 -0
  644. package/tutorial/multi_agents/sales_agent.ts +88 -0
  645. package/tutorial/multi_agents/sales_agent_with_search.ts +116 -0
  646. package/tutorial/multi_agents/sales_knowledge.md +139 -0
  647. package/tutorial/multi_agents/support_knowledge.md +256 -0
  648. package/dist/cli/mock-data.d.ts +0 -37
  649. package/dist/cli/mock-data.js +0 -80
  650. package/dist/cli/mock-data.js.map +0 -1
  651. package/dist/generateVerbTypes.d.ts +0 -11
  652. package/dist/generateVerbTypes.js +0 -265
  653. package/dist/generateVerbTypes.js.map +0 -1
  654. package/dist/rest/namespaces/compat.d.ts +0 -585
  655. package/dist/rest/namespaces/compat.js +0 -735
  656. package/dist/rest/namespaces/compat.js.map +0 -1
@@ -11,23 +11,16 @@
11
11
  *
12
12
  * Multiple instances are supported; use `tool_name` in config to differentiate.
13
13
  */
14
- import { SkillBase } from '../SkillBase.js';
14
+ import { SkillBase, defineSkillTool } from '../SkillBase.js';
15
15
  import { FunctionResult } from '../../FunctionResult.js';
16
+ import { _publicFetch, _RedirectRefused } from '../../PublicFetch.js';
17
+ import { _RobotsRules } from '../../RobotsTxt.js';
16
18
  import { resolveAndValidateUrl, validateUrl, MAX_SKILL_INPUT_LENGTH } from '../../SecurityUtils.js';
17
19
  import { getLogger } from '../../Logger.js';
18
- import * as cheerio from 'cheerio';
19
20
  const log = getLogger('SpiderSkill');
20
21
  const WHITESPACE_REGEX = /\s+/g;
21
- /** Tags stripped from HTML before text extraction. */
22
- const STRIP_TAG_REGEXES = [
23
- /<script\b[^>]*>[\s\S]*?<\/script>/gi,
24
- /<style\b[^>]*>[\s\S]*?<\/style>/gi,
25
- /<nav\b[^>]*>[\s\S]*?<\/nav>/gi,
26
- /<header\b[^>]*>[\s\S]*?<\/header>/gi,
27
- /<footer\b[^>]*>[\s\S]*?<\/footer>/gi,
28
- /<aside\b[^>]*>[\s\S]*?<\/aside>/gi,
29
- /<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi,
30
- ];
22
+ /** Headers that carry credentials for one origin (as _publicFetch treats them). */
23
+ const ORIGIN_BOUND_HEADERS = new Set(['authorization', 'cookie', 'proxy-authorization']);
31
24
  /**
32
25
  * Fast web scraping skill optimized for speed and token efficiency.
33
26
  *
@@ -39,7 +32,9 @@ const STRIP_TAG_REGEXES = [
39
32
  *
40
33
  * @example
41
34
  * ```ts
42
- * agent.addSkill('spider', { max_pages: 5, max_depth: 2 });
35
+ * import { AgentBase } from '@signalwire/sdk';
36
+ * const agent = new AgentBase({ name: 'demo', route: '/' });
37
+ * await agent.addSkillByName('spider', { max_pages: 5, max_depth: 2 });
43
38
  * ```
44
39
  */
45
40
  export class SpiderSkill extends SkillBase {
@@ -51,20 +46,64 @@ export class SpiderSkill extends SkillBase {
51
46
  static REQUIRED_PACKAGES = ['cheerio'];
52
47
  static REQUIRED_ENV_VARS = [];
53
48
  static SUPPORTS_MULTIPLE_INSTANCES = true;
49
+ /**
50
+ * Each setting's default. The parameter schema and setup() both read this,
51
+ * so the schema can't advertise a default the skill doesn't use.
52
+ */
53
+ static DEFAULTS = {
54
+ delay: 0.1,
55
+ concurrent_requests: 5,
56
+ timeout: 5,
57
+ max_pages: 1,
58
+ max_depth: 0,
59
+ extract_type: 'fast_text',
60
+ max_text_length: 3000,
61
+ clean_text: true,
62
+ cache_enabled: true,
63
+ follow_robots_txt: false,
64
+ user_agent: 'Spider/1.0 (SignalWire AI Agent)',
65
+ };
66
+ /** The extraction methods scrape_url implements. */
67
+ static EXTRACT_TYPES = [
68
+ 'fast_text',
69
+ 'markdown',
70
+ 'structured',
71
+ ];
72
+ /**
73
+ * Values the schema once listed that were never implemented. They have
74
+ * always worked as fast_text, so they still do, with a warning.
75
+ */
76
+ static LEGACY_EXTRACT_TYPES = new Set([
77
+ 'clean_text',
78
+ 'full_text',
79
+ 'html',
80
+ 'custom',
81
+ ]);
82
+ /** Seconds to keep a site's robots.txt rules; RFC 9309 allows up to 24 hours. */
83
+ static ROBOTS_TTL = 24 * 60 * 60;
54
84
  static getParameterSchema() {
85
+ const d = SpiderSkill.DEFAULTS;
55
86
  return {
56
87
  ...super.getParameterSchema(),
88
+ // The base schema's default is the skill name, but tool_name is a
89
+ // prefix that's empty when unset (Python skill.py:306 too).
90
+ tool_name: {
91
+ type: 'string',
92
+ description: 'Prefix for the tool names, as in <tool_name>_scrape_url, and for the instance key. ' +
93
+ 'Unset, the tools keep their plain names.',
94
+ required: false,
95
+ },
57
96
  delay: {
58
97
  type: 'number',
59
98
  description: 'Delay between requests in seconds',
60
- default: 0.1,
99
+ default: d.delay,
61
100
  required: false,
62
101
  min: 0,
63
102
  },
64
103
  concurrent_requests: {
65
104
  type: 'integer',
66
- description: 'Number of concurrent requests allowed',
67
- default: 5,
105
+ description: 'Deprecated, and has no effect: the spider fetches one page at a time',
106
+ default: d.concurrent_requests,
68
107
  required: false,
69
108
  min: 1,
70
109
  max: 20,
@@ -72,7 +111,7 @@ export class SpiderSkill extends SkillBase {
72
111
  timeout: {
73
112
  type: 'integer',
74
113
  description: 'Request timeout in seconds',
75
- default: 5,
114
+ default: d.timeout,
76
115
  required: false,
77
116
  min: 1,
78
117
  max: 60,
@@ -80,7 +119,7 @@ export class SpiderSkill extends SkillBase {
80
119
  max_pages: {
81
120
  type: 'integer',
82
121
  description: 'Maximum number of pages to scrape',
83
- default: 1,
122
+ default: d.max_pages,
84
123
  required: false,
85
124
  min: 1,
86
125
  max: 100,
@@ -88,7 +127,7 @@ export class SpiderSkill extends SkillBase {
88
127
  max_depth: {
89
128
  type: 'integer',
90
129
  description: 'Maximum crawl depth (0 = single page only)',
91
- default: 0,
130
+ default: d.max_depth,
92
131
  required: false,
93
132
  min: 0,
94
133
  max: 5,
@@ -96,14 +135,14 @@ export class SpiderSkill extends SkillBase {
96
135
  extract_type: {
97
136
  type: 'string',
98
137
  description: 'Content extraction method',
99
- default: 'fast_text',
138
+ default: d.extract_type,
100
139
  required: false,
101
- enum: ['fast_text', 'clean_text', 'full_text', 'html', 'markdown', 'structured', 'custom'],
140
+ enum: [...SpiderSkill.EXTRACT_TYPES],
102
141
  },
103
142
  max_text_length: {
104
143
  type: 'integer',
105
144
  description: 'Maximum text length to return',
106
- default: 3000,
145
+ default: d.max_text_length,
107
146
  required: false,
108
147
  min: 100,
109
148
  max: 100000,
@@ -111,7 +150,7 @@ export class SpiderSkill extends SkillBase {
111
150
  clean_text: {
112
151
  type: 'boolean',
113
152
  description: 'Whether to clean extracted text',
114
- default: true,
153
+ default: d.clean_text,
115
154
  required: false,
116
155
  },
117
156
  selectors: {
@@ -131,7 +170,7 @@ export class SpiderSkill extends SkillBase {
131
170
  user_agent: {
132
171
  type: 'string',
133
172
  description: 'User agent string for requests',
134
- default: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
173
+ default: d.user_agent,
135
174
  required: false,
136
175
  },
137
176
  headers: {
@@ -143,63 +182,72 @@ export class SpiderSkill extends SkillBase {
143
182
  },
144
183
  follow_robots_txt: {
145
184
  type: 'boolean',
146
- // Matches Python's __init__ runtime fallback (skills/spider/skill.py:177
147
- // uses `params.get('follow_robots_txt', False)`). Python's schema said
148
- // True but the runtime and TS both use False — aligning the schema
149
- // eliminates the cross-SDK contract mismatch.
150
- description: 'Whether to respect robots.txt (default: false to match Python runtime)',
151
- default: false,
185
+ description: "Skip pages that the site's robots.txt disallows for user_agent",
186
+ default: d.follow_robots_txt,
152
187
  required: false,
153
188
  },
154
189
  cache_enabled: {
155
190
  type: 'boolean',
156
191
  description: 'Whether to cache scraped pages',
157
- default: true,
192
+ default: d.cache_enabled,
158
193
  required: false,
159
194
  },
160
195
  };
161
196
  }
162
197
  // Runtime state, populated in setup()
163
- delay = 0.1;
164
- concurrentRequests = 5;
165
- timeout = 5;
166
- maxPages = 1;
167
- maxDepth = 0;
168
- extractType = 'fast_text';
169
- maxTextLength = 3000;
170
- cleanText = true;
171
- cacheEnabled = true;
172
- followRobotsTxt = true;
173
- userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
198
+ delay = SpiderSkill.DEFAULTS.delay;
199
+ concurrentRequests = SpiderSkill.DEFAULTS.concurrent_requests;
200
+ timeout = SpiderSkill.DEFAULTS.timeout;
201
+ maxPages = SpiderSkill.DEFAULTS.max_pages;
202
+ maxDepth = SpiderSkill.DEFAULTS.max_depth;
203
+ extractType = SpiderSkill.DEFAULTS.extract_type;
204
+ maxTextLength = SpiderSkill.DEFAULTS.max_text_length;
205
+ cleanText = SpiderSkill.DEFAULTS.clean_text;
206
+ cacheEnabled = SpiderSkill.DEFAULTS.cache_enabled;
207
+ followRobotsTxt = SpiderSkill.DEFAULTS.follow_robots_txt;
208
+ userAgent = SpiderSkill.DEFAULTS.user_agent;
174
209
  headers = {};
175
210
  selectors = {};
176
211
  compiledFollowPatterns = [];
177
212
  cache = null;
178
213
  cacheMaxSize = 100;
214
+ /** robots.txt rules per origin, with when they expire, when follow_robots_txt is on. */
215
+ robots = new Map();
216
+ // Lazily-loaded optional `cheerio` module, populated in setup(). Non-null
217
+ // for the lifetime of an initialized skill (setup() returns false if absent).
218
+ _cheerio;
179
219
  getInstanceKey() {
180
220
  const toolName = this.getConfig('tool_name', this.skillName);
181
221
  return `${this.skillName}_${toolName}`;
182
222
  }
183
223
  async setup() {
224
+ // Load the optional cheerio dependency once. If it isn't installed, the
225
+ // skill cannot scrape — fail setup cleanly (validatePackages() also flags
226
+ // it). Cached on the instance so the sync extract helpers can use it.
227
+ try {
228
+ this._cheerio = await import('cheerio');
229
+ }
230
+ catch {
231
+ log.error('spider: the optional "cheerio" package is required for this skill but is not installed');
232
+ return false;
233
+ }
234
+ const d = SpiderSkill.DEFAULTS;
184
235
  // Performance
185
- this.delay = this.getConfig('delay', 0.1);
186
- this.concurrentRequests = this.getConfig('concurrent_requests', 5);
187
- this.timeout = this.getConfig('timeout', 5);
236
+ this.delay = this.getConfig('delay', d.delay);
237
+ this.concurrentRequests = this.getConfig('concurrent_requests', d.concurrent_requests);
238
+ this.timeout = this.getConfig('timeout', d.timeout);
188
239
  // Crawl limits
189
- this.maxPages = this.getConfig('max_pages', 1);
190
- this.maxDepth = this.getConfig('max_depth', 0);
240
+ this.maxPages = this.getConfig('max_pages', d.max_pages);
241
+ this.maxDepth = this.getConfig('max_depth', d.max_depth);
191
242
  // Content processing
192
- this.extractType = this.getConfig('extract_type', 'fast_text');
193
- // Python has an internal inconsistency (schema: 10000, __init__ fallback:
194
- // 3000). The effective runtime default is 3000 because Python reads via
195
- // `self.params.get('max_text_length', 3000)` and schema defaults are not
196
- // applied. Use 3000 for parity. Follow-up: reconcile Python schema vs init.
197
- this.maxTextLength = this.getConfig('max_text_length', 3000);
198
- this.cleanText = this.getConfig('clean_text', true);
243
+ this.extractType = this.getConfig('extract_type', d.extract_type);
244
+ this.maxTextLength = this.getConfig('max_text_length', d.max_text_length);
245
+ this.cleanText = this.getConfig('clean_text', d.clean_text);
199
246
  // Features
200
- this.cacheEnabled = this.getConfig('cache_enabled', true);
201
- this.followRobotsTxt = this.getConfig('follow_robots_txt', false);
202
- this.userAgent = this.getConfig('user_agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
247
+ this.cacheEnabled = this.getConfig('cache_enabled', d.cache_enabled);
248
+ this.followRobotsTxt = this.getConfig('follow_robots_txt', d.follow_robots_txt);
249
+ this.userAgent = this.getConfig('user_agent', d.user_agent);
250
+ this.robots = new Map();
203
251
  // Optional headers + user-agent merge
204
252
  const headers = this.getConfig('headers', {}) ?? {};
205
253
  this.headers = { ...headers, 'User-Agent': this.userAgent };
@@ -213,7 +261,22 @@ export class SpiderSkill extends SkillBase {
213
261
  return false;
214
262
  }
215
263
  if (this.concurrentRequests < 1 || this.concurrentRequests > 20) {
216
- log.error('spider: concurrent_requests must be between 1 and 20', { concurrent_requests: this.concurrentRequests });
264
+ log.error('spider: concurrent_requests must be between 1 and 20', {
265
+ concurrent_requests: this.concurrentRequests,
266
+ });
267
+ return false;
268
+ }
269
+ if ('concurrent_requests' in this.config) {
270
+ log.warn('spider: concurrent_requests is deprecated and has no effect: the spider fetches one page at a time');
271
+ }
272
+ // Validate the extraction method
273
+ const types = SpiderSkill.EXTRACT_TYPES.join(', ');
274
+ if (SpiderSkill.LEGACY_EXTRACT_TYPES.has(this.extractType)) {
275
+ log.warn(`spider: extract_type '${this.extractType}' was never implemented and works as fast_text; use one of ${types}`);
276
+ this.extractType = 'fast_text';
277
+ }
278
+ else if (!SpiderSkill.EXTRACT_TYPES.includes(this.extractType)) {
279
+ log.error(`spider: unknown extract_type '${this.extractType}'; use one of ${types}`);
217
280
  return false;
218
281
  }
219
282
  if (this.maxPages < 1) {
@@ -267,8 +330,11 @@ export class SpiderSkill extends SkillBase {
267
330
  getTools() {
268
331
  const toolPrefix = this.getConfig('tool_name', '');
269
332
  const prefix = toolPrefix ? `${toolPrefix}_` : '';
333
+ // The three handlers keep their defensive typeof/.trim()/URL-validation
334
+ // narrowing in the private methods: `url`/`start_url` are required, but the
335
+ // model can still emit an empty or wrong-typed value.
270
336
  return [
271
- {
337
+ defineSkillTool({
272
338
  name: `${prefix}scrape_url`,
273
339
  description: 'Extract text content from a single web page',
274
340
  parameters: {
@@ -279,8 +345,8 @@ export class SpiderSkill extends SkillBase {
279
345
  },
280
346
  required: ['url'],
281
347
  handler: async (args, rawData) => this._scrapeUrlHandler(args, rawData),
282
- },
283
- {
348
+ }),
349
+ defineSkillTool({
284
350
  name: `${prefix}crawl_site`,
285
351
  description: 'Crawl multiple pages starting from a URL',
286
352
  parameters: {
@@ -291,8 +357,8 @@ export class SpiderSkill extends SkillBase {
291
357
  },
292
358
  required: ['start_url'],
293
359
  handler: async (args, rawData) => this._crawlSiteHandler(args, rawData),
294
- },
295
- {
360
+ }),
361
+ defineSkillTool({
296
362
  name: `${prefix}extract_structured_data`,
297
363
  description: 'Extract specific data from a web page using selectors',
298
364
  parameters: {
@@ -303,18 +369,30 @@ export class SpiderSkill extends SkillBase {
303
369
  },
304
370
  required: ['url'],
305
371
  handler: async (args, rawData) => this._extractStructuredHandler(args, rawData),
306
- },
372
+ }),
307
373
  ];
308
374
  }
309
375
  // ---------------------------------------------------------------------------
310
376
  // Internal helpers
311
377
  // ---------------------------------------------------------------------------
312
378
  /**
313
- * Redirect a target URL through `SPIDER_BASE_URL` when set. Used by the
314
- * porting-sdk's `audit_skills_dispatch.py` to point every outbound fetch
315
- * at its loopback fixture without requiring the test URL to resolve.
379
+ * Redirect a target URL through `SPIDER_BASE_URL` when set. Used by tests
380
+ * to point every outbound fetch at a loopback fixture without requiring
381
+ * the test URL to resolve.
316
382
  * Preserves the path (and any query/fragment) from the original target.
317
383
  */
384
+ /** Whether a URL is on the operator-set SPIDER_BASE_URL's origin, a trusted target. */
385
+ static _isAuditOrigin(target) {
386
+ const base = process.env['SPIDER_BASE_URL'];
387
+ if (!base)
388
+ return false;
389
+ try {
390
+ return new URL(target).origin === new URL(base).origin;
391
+ }
392
+ catch {
393
+ return false;
394
+ }
395
+ }
318
396
  static _redirectForAudit(target) {
319
397
  const base = process.env['SPIDER_BASE_URL'];
320
398
  if (!base)
@@ -331,6 +409,94 @@ export class SpiderSkill extends SkillBase {
331
409
  return `${base.replace(/\/+$/, '')}${path}`;
332
410
  }
333
411
  }
412
+ /**
413
+ * Whether a URL the caller supplied may be fetched: it must not be private
414
+ * or internal (`SWML_ALLOW_PRIVATE_URLS` allows it). With `SPIDER_BASE_URL`
415
+ * set, the fetch goes to that operator-configured base instead of the URL's
416
+ * host, so the host isn't checked.
417
+ */
418
+ static async _targetAllowed(url) {
419
+ if (SpiderSkill._redirectForAudit(url) !== url)
420
+ return true;
421
+ try {
422
+ await resolveAndValidateUrl(url);
423
+ return true;
424
+ }
425
+ catch {
426
+ return false;
427
+ }
428
+ }
429
+ /**
430
+ * False when follow_robots_txt is on and the site's robots.txt disallows
431
+ * `url` for user_agent.
432
+ *
433
+ * As in Python's `urllib.robotparser`, a robots.txt answered with 401 or
434
+ * 403 disallows everything and any other 4xx allows everything. Rules are
435
+ * kept for {@link ROBOTS_TTL}. A server error or failed request disallows
436
+ * this page without keeping anything, so the next request tries again.
437
+ */
438
+ async _allowedByRobots(url, opts = {}) {
439
+ if (!this.followRobotsTxt)
440
+ return true;
441
+ let origin;
442
+ try {
443
+ // Check the URL as it will be requested: fetch normalizes it, so
444
+ // /public/../private is a request for /private.
445
+ const parsed = new URL(url);
446
+ url = parsed.href;
447
+ origin = parsed.origin;
448
+ }
449
+ catch {
450
+ return false;
451
+ }
452
+ const now = performance.now() / 1000;
453
+ const cached = this.robots.get(origin);
454
+ if (cached && cached.expires > now)
455
+ return cached.rules.canFetch(this.userAgent, url);
456
+ const robotsUrl = `${origin}/robots.txt`;
457
+ const fetchUrl = SpiderSkill._redirectForAudit(robotsUrl);
458
+ // Credentials for the page's origin don't go to another origin's
459
+ // robots.txt, as _publicFetch drops them on a redirect there.
460
+ const headers = { ...this.headers };
461
+ if (opts.withoutCredentials) {
462
+ for (const name of Object.keys(headers)) {
463
+ if (ORIGIN_BOUND_HEADERS.has(name.toLowerCase()))
464
+ delete headers[name];
465
+ }
466
+ }
467
+ let status;
468
+ let body = '';
469
+ const controller = new AbortController();
470
+ const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
471
+ try {
472
+ const response = await _publicFetch(fetchUrl, {
473
+ method: 'GET',
474
+ headers,
475
+ signal: controller.signal,
476
+ allowPrivate: fetchUrl !== robotsUrl || SpiderSkill._isAuditOrigin(robotsUrl),
477
+ });
478
+ status = response.status;
479
+ body = await response.text();
480
+ }
481
+ catch {
482
+ status = 599;
483
+ }
484
+ finally {
485
+ clearTimeout(timer);
486
+ }
487
+ // Unavailable for now: disallow this request, and try again next time.
488
+ if (status >= 500)
489
+ return false;
490
+ let rules;
491
+ if (status === 401 || status === 403)
492
+ rules = _RobotsRules.disallowAll();
493
+ else if (status >= 400)
494
+ rules = _RobotsRules.allowAll();
495
+ else
496
+ rules = new _RobotsRules(body);
497
+ this.robots.set(origin, { rules, expires: now + SpiderSkill.ROBOTS_TTL });
498
+ return rules.canFetch(this.userAgent, url);
499
+ }
334
500
  /** Fetch a URL with caching and timeout handling. Returns null on failure. */
335
501
  async _fetchUrl(url) {
336
502
  if (this.cacheEnabled && this.cache?.has(url)) {
@@ -342,13 +508,29 @@ export class SpiderSkill extends SkillBase {
342
508
  // a `https://audit.example/page` target and expects the skill to hit
343
509
  // `http://127.0.0.1:NNNN/page` — preserving the path after the host.
344
510
  const fetchUrl = SpiderSkill._redirectForAudit(url);
511
+ const startOrigin = new URL(fetchUrl).origin;
512
+ let crossedOrigin = false;
345
513
  const controller = new AbortController();
346
514
  const timer = setTimeout(() => controller.abort(), this.timeout * 1000);
347
515
  try {
348
- const response = await fetch(fetchUrl, {
516
+ // _publicFetch checks the URL and every redirect, and refuses a
517
+ // connection to a private or internal address. An operator-set
518
+ // SPIDER_BASE_URL is a trusted target, so its fetches skip the check.
519
+ const response = await _publicFetch(fetchUrl, {
349
520
  method: 'GET',
350
521
  headers: this.headers,
351
522
  signal: controller.signal,
523
+ allowPrivate: fetchUrl !== url,
524
+ // Each redirect's target needs its own robots.txt check.
525
+ allowRedirect: this.followRobotsTxt
526
+ ? (target) => {
527
+ // Once a redirect leaves the page's origin, _publicFetch has
528
+ // dropped the credentials for good, and so does robots.txt.
529
+ if (new URL(target).origin !== startOrigin)
530
+ crossedOrigin = true;
531
+ return this._allowedByRobots(target, { withoutCredentials: crossedOrigin });
532
+ }
533
+ : undefined,
352
534
  });
353
535
  if (!response.ok) {
354
536
  log.error('spider: fetch failed', { url, status: response.status });
@@ -371,6 +553,10 @@ export class SpiderSkill extends SkillBase {
371
553
  return cached;
372
554
  }
373
555
  catch (err) {
556
+ if (err instanceof _RedirectRefused) {
557
+ log.info('spider: robots.txt disallows the redirect', { url, target: err.target });
558
+ return null;
559
+ }
374
560
  log.error('spider: fetch error', {
375
561
  url,
376
562
  error: err instanceof Error ? err.message : String(err),
@@ -411,18 +597,10 @@ export class SpiderSkill extends SkillBase {
411
597
  // decoded (Python's lxml does this natively). Previously TS only
412
598
  // decoded six hand-coded entities, leaving `&mdash;`, `&#8212;`, etc.
413
599
  // literal in the output.
414
- const $ = cheerio.load(body);
600
+ const $ = this._cheerio.load(body);
415
601
  // Strip noise tags before extracting text — matches Python's
416
602
  // lxml drop_tree() on the same 7 tags.
417
- for (const tag of [
418
- 'script',
419
- 'style',
420
- 'nav',
421
- 'header',
422
- 'footer',
423
- 'aside',
424
- 'noscript',
425
- ]) {
603
+ for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside', 'noscript']) {
426
604
  $(tag).remove();
427
605
  }
428
606
  let text = $('body').text();
@@ -438,9 +616,7 @@ export class SpiderSkill extends SkillBase {
438
616
  const keepStart = Math.floor((this.maxTextLength * 2) / 3);
439
617
  const keepEnd = Math.floor(this.maxTextLength / 3);
440
618
  text =
441
- text.slice(0, keepStart) +
442
- '\n\n[...CONTENT TRUNCATED...]\n\n' +
443
- text.slice(-keepEnd);
619
+ text.slice(0, keepStart) + '\n\n[...CONTENT TRUNCATED...]\n\n' + text.slice(-keepEnd);
444
620
  }
445
621
  return text;
446
622
  }
@@ -483,7 +659,7 @@ export class SpiderSkill extends SkillBase {
483
659
  /** Markdown extraction using cheerio. Mirrors Python `_markdown_extract`. */
484
660
  _markdownExtract(response) {
485
661
  try {
486
- const $ = cheerio.load(response.body);
662
+ const $ = this._cheerio.load(response.body);
487
663
  // Remove unwanted tags
488
664
  for (const tag of ['script', 'style', 'nav', 'header', 'footer', 'aside']) {
489
665
  $(tag).remove();
@@ -527,7 +703,7 @@ export class SpiderSkill extends SkillBase {
527
703
  _structuredExtract(cached, selectors) {
528
704
  let $;
529
705
  try {
530
- $ = cheerio.load(cached.body);
706
+ $ = this._cheerio.load(cached.body);
531
707
  }
532
708
  catch (err) {
533
709
  // Python returns `{'error': str(e)}` from _structured_extract when the
@@ -585,13 +761,12 @@ export class SpiderSkill extends SkillBase {
585
761
  return new FunctionResult(`Invalid URL: ${url}`);
586
762
  }
587
763
  // SSRF protection
588
- const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
589
- try {
590
- await resolveAndValidateUrl(url, allowPrivate);
591
- }
592
- catch {
764
+ if (!(await SpiderSkill._targetAllowed(url))) {
593
765
  return new FunctionResult('URL rejected: cannot access private or internal URLs');
594
766
  }
767
+ if (!(await this._allowedByRobots(url))) {
768
+ return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
769
+ }
595
770
  const cached = await this._fetchUrl(url);
596
771
  if (!cached) {
597
772
  return new FunctionResult(`Failed to fetch ${url}`);
@@ -628,11 +803,7 @@ export class SpiderSkill extends SkillBase {
628
803
  if (!/^https?:\/\//.test(startUrl)) {
629
804
  return new FunctionResult(`Invalid URL: ${startUrl}`);
630
805
  }
631
- const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
632
- try {
633
- await resolveAndValidateUrl(startUrl, allowPrivate);
634
- }
635
- catch {
806
+ if (!(await SpiderSkill._targetAllowed(startUrl))) {
636
807
  return new FunctionResult('URL rejected: cannot access private or internal URLs');
637
808
  }
638
809
  const maxDepth = this.maxDepth;
@@ -642,6 +813,7 @@ export class SpiderSkill extends SkillBase {
642
813
  if (maxPages < 1)
643
814
  return new FunctionResult('Max pages must be at least 1');
644
815
  const visited = new Set();
816
+ const disallowed = new Set();
645
817
  const toVisit = [[startUrl, 0]];
646
818
  const results = [];
647
819
  let startHost;
@@ -656,8 +828,13 @@ export class SpiderSkill extends SkillBase {
656
828
  if (!next)
657
829
  break;
658
830
  const [url, depth] = next;
659
- if (visited.has(url) || depth > maxDepth)
831
+ if (visited.has(url) || disallowed.has(url) || depth > maxDepth)
832
+ continue;
833
+ if (!(await this._allowedByRobots(url))) {
834
+ log.info('spider: robots.txt disallows a page; skipping it', { url });
835
+ disallowed.add(url);
660
836
  continue;
837
+ }
661
838
  const cached = await this._fetchUrl(url);
662
839
  if (!cached)
663
840
  continue;
@@ -677,7 +854,7 @@ export class SpiderSkill extends SkillBase {
677
854
  // xpath('//a[@href]/@href') which has the same robustness).
678
855
  if (depth < maxDepth) {
679
856
  try {
680
- const $links = cheerio.load(cached.body);
857
+ const $links = this._cheerio.load(cached.body);
681
858
  const hrefs = [];
682
859
  $links('a[href]').each((_, el) => {
683
860
  const href = $links(el).attr('href');
@@ -733,10 +910,7 @@ export class SpiderSkill extends SkillBase {
733
910
  if (results.length === 0) {
734
911
  return new FunctionResult(`No pages could be crawled from ${startUrl}`);
735
912
  }
736
- const lines = [
737
- `Crawled ${results.length} pages from ${startHost}:`,
738
- '',
739
- ];
913
+ const lines = [`Crawled ${results.length} pages from ${startHost}:`, ''];
740
914
  results.forEach((r, i) => {
741
915
  lines.push(`${i + 1}. ${r.url} (depth: ${r.depth}, ${r.contentLength} chars)`);
742
916
  lines.push(` Summary: ${r.summary.slice(0, 100)}...`);
@@ -756,16 +930,15 @@ export class SpiderSkill extends SkillBase {
756
930
  if (!/^https?:\/\//.test(url)) {
757
931
  return new FunctionResult(`Invalid URL: ${url}`);
758
932
  }
759
- const allowPrivate = process.env['SWML_ALLOW_PRIVATE_URLS'] === 'true';
760
- try {
761
- await resolveAndValidateUrl(url, allowPrivate);
762
- }
763
- catch {
933
+ if (!(await SpiderSkill._targetAllowed(url))) {
764
934
  return new FunctionResult('URL rejected: cannot access private or internal URLs');
765
935
  }
766
936
  if (!this.selectors || Object.keys(this.selectors).length === 0) {
767
937
  return new FunctionResult('No selectors configured for structured data extraction');
768
938
  }
939
+ if (!(await this._allowedByRobots(url))) {
940
+ return new FunctionResult(`The site's robots.txt disallows fetching ${url}`);
941
+ }
769
942
  const cached = await this._fetchUrl(url);
770
943
  if (!cached) {
771
944
  return new FunctionResult(`Failed to fetch ${url}`);