@signalwire/sdk 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (325) hide show
  1. package/.github/workflows/publish.yml +27 -0
  2. package/.github/workflows/test.yml +17 -0
  3. package/CLAUDE.md +112 -0
  4. package/README.md +254 -0
  5. package/docs/MIGRATION-2.0.md +81 -0
  6. package/docs/agent-guide.md +869 -0
  7. package/docs/agent_guide.md +1026 -0
  8. package/docs/api-reference.md +2420 -0
  9. package/docs/api_reference.md +1301 -0
  10. package/docs/architecture.md +843 -0
  11. package/docs/cli-guide.md +579 -0
  12. package/docs/cli_guide.md +448 -0
  13. package/docs/cloud_functions_guide.md +474 -0
  14. package/docs/configuration.md +422 -0
  15. package/docs/contexts-guide.md +787 -0
  16. package/docs/contexts_guide.md +511 -0
  17. package/docs/datamap-guide.md +1141 -0
  18. package/docs/datamap_guide.md +595 -0
  19. package/docs/llm_parameters.md +239 -0
  20. package/docs/mcp_gateway_reference.md +648 -0
  21. package/docs/mcp_integration.md +108 -0
  22. package/docs/prefabs-guide.md +800 -0
  23. package/docs/sdk_features.md +477 -0
  24. package/docs/security.md +436 -0
  25. package/docs/serverless-guide.md +513 -0
  26. package/docs/skills-guide.md +857 -0
  27. package/docs/skills-system.md +301 -0
  28. package/docs/skills_parameter_schema.md +392 -0
  29. package/docs/skills_system.md +467 -0
  30. package/docs/swaig-reference.md +1627 -0
  31. package/docs/swaig_reference.md +910 -0
  32. package/docs/swml_service_guide.md +600 -0
  33. package/docs/third_party_skills.md +533 -0
  34. package/docs/web_service.md +677 -0
  35. package/examples/README.md +131 -0
  36. package/examples/advanced-datamap.ts +100 -0
  37. package/examples/advanced-dynamic-config.ts +70 -0
  38. package/examples/auto-vivified.ts +94 -0
  39. package/examples/call-flow.ts +63 -0
  40. package/examples/comprehensive-dynamic.ts +125 -0
  41. package/examples/contexts-steps.ts +65 -0
  42. package/examples/custom-path.ts +45 -0
  43. package/examples/datamap-tools.ts +55 -0
  44. package/examples/datasphere-multi-instance.ts +63 -0
  45. package/examples/datasphere-serverless-env.ts +68 -0
  46. package/examples/datasphere-webhook-env.ts +66 -0
  47. package/examples/datasphere.ts +37 -0
  48. package/examples/declarative.ts +100 -0
  49. package/examples/dynamic-config.ts +54 -0
  50. package/examples/dynamic-info-gatherer.ts +69 -0
  51. package/examples/dynamic-swml-service.ts +67 -0
  52. package/examples/gather-info.ts +71 -0
  53. package/examples/gather-per-question-functions-demo.ts +140 -0
  54. package/examples/joke-agent.ts +34 -0
  55. package/examples/kubernetes-agent.ts +45 -0
  56. package/examples/llm-params.ts +69 -0
  57. package/examples/mcp-agent.ts +84 -0
  58. package/examples/mcp-gateway.ts +35 -0
  59. package/examples/multi-agent.ts +73 -0
  60. package/examples/multi-endpoint.ts +79 -0
  61. package/examples/pom-prompt.ts +71 -0
  62. package/examples/prefab-concierge.ts +47 -0
  63. package/examples/prefab-faq.ts +54 -0
  64. package/examples/prefab-info-gatherer.ts +33 -0
  65. package/examples/prefab-receptionist.ts +36 -0
  66. package/examples/prefab-survey.ts +68 -0
  67. package/examples/record-call.ts +67 -0
  68. package/examples/relay-demo.ts +33 -0
  69. package/examples/room-and-sip.ts +94 -0
  70. package/examples/serverless-lambda.ts +35 -0
  71. package/examples/session-state.ts +83 -0
  72. package/examples/simple-agent.ts +36 -0
  73. package/examples/simple-static.ts +29 -0
  74. package/examples/skills-demo.ts +38 -0
  75. package/examples/step-function-inheritance-demo.ts +124 -0
  76. package/examples/swaig-features.ts +187 -0
  77. package/examples/swml-service-routing.ts +66 -0
  78. package/examples/swml-service.ts +38 -0
  79. package/examples/tap.ts +68 -0
  80. package/examples/verb-methods.ts +45 -0
  81. package/examples/web-search-multi-instance.ts +72 -0
  82. package/examples/web-search.ts +38 -0
  83. package/examples/wikipedia.ts +36 -0
  84. package/livewire/README.md +140 -0
  85. package/livewire/docs/migration-guide.md +228 -0
  86. package/livewire/examples/livewire-basic-agent.ts +74 -0
  87. package/livewire/examples/livewire-handoff.ts +197 -0
  88. package/livewire/examples/livewire-multi-tool.ts +117 -0
  89. package/package.json +38 -0
  90. package/relay/README.md +70 -0
  91. package/relay/RELAY_IMPLEMENTATION_GUIDE.md +1257 -0
  92. package/relay/docs/call-methods.md +474 -0
  93. package/relay/docs/client-reference.md +142 -0
  94. package/relay/docs/events.md +134 -0
  95. package/relay/docs/getting-started.md +134 -0
  96. package/relay/docs/guide.md +306 -0
  97. package/relay/docs/messaging.md +182 -0
  98. package/relay/examples/relay-inbound.ts +66 -0
  99. package/relay/examples/relay-messaging.ts +71 -0
  100. package/relay/examples/relay-outbound.ts +76 -0
  101. package/rest/README.md +95 -0
  102. package/rest/docs/calling.md +273 -0
  103. package/rest/docs/client-reference.md +107 -0
  104. package/rest/docs/compat.md +204 -0
  105. package/rest/docs/fabric.md +183 -0
  106. package/rest/docs/getting-started.md +110 -0
  107. package/rest/docs/guide.md +351 -0
  108. package/rest/docs/namespaces.md +285 -0
  109. package/rest/examples/rest-10dlc-registration.ts +161 -0
  110. package/rest/examples/rest-calling-ivr-and-ai.ts +102 -0
  111. package/rest/examples/rest-calling-play-and-record.ts +91 -0
  112. package/rest/examples/rest-client.ts +65 -0
  113. package/rest/examples/rest-compat-laml.ts +210 -0
  114. package/rest/examples/rest-datasphere-search.ts +82 -0
  115. package/rest/examples/rest-fabric-conferences-and-routing.ts +144 -0
  116. package/rest/examples/rest-fabric-subscribers-and-sip.ts +108 -0
  117. package/rest/examples/rest-fabric-swml-and-callflows.ts +99 -0
  118. package/rest/examples/rest-manage-resources.ts +65 -0
  119. package/rest/examples/rest-phone-number-management.ts +197 -0
  120. package/rest/examples/rest-queues-mfa-and-recordings.ts +143 -0
  121. package/rest/examples/rest-video-rooms.ts +186 -0
  122. package/src/AgentBase.ts +2489 -0
  123. package/src/AgentServer.ts +415 -0
  124. package/src/AuthHandler.ts +256 -0
  125. package/src/ConfigLoader.ts +447 -0
  126. package/src/ContextBuilder.ts +1150 -0
  127. package/src/DataMap.ts +449 -0
  128. package/src/FunctionResult.ts +866 -0
  129. package/src/Logger.ts +337 -0
  130. package/src/PomBuilder.ts +439 -0
  131. package/src/PromptManager.ts +133 -0
  132. package/src/SWMLHandler.ts +298 -0
  133. package/src/SWMLService.ts +717 -0
  134. package/src/SchemaUtils.ts +361 -0
  135. package/src/SecurityUtils.ts +153 -0
  136. package/src/ServerlessAdapter.ts +221 -0
  137. package/src/SessionManager.ts +311 -0
  138. package/src/SslConfig.ts +118 -0
  139. package/src/SwaigFunction.ts +276 -0
  140. package/src/SwmlBuilder.ts +258 -0
  141. package/src/SwmlVerbMethods.generated.ts +234 -0
  142. package/src/TypeInference.ts +181 -0
  143. package/src/WebService.ts +619 -0
  144. package/src/cli/agent-loader.ts +166 -0
  145. package/src/cli/mock-data.ts +111 -0
  146. package/src/cli/swaig-test.ts +452 -0
  147. package/src/generateVerbTypes.ts +322 -0
  148. package/src/index.ts +167 -0
  149. package/src/livewire/index.ts +950 -0
  150. package/src/prefabs/ConciergeAgent.ts +287 -0
  151. package/src/prefabs/FAQBotAgent.ts +384 -0
  152. package/src/prefabs/InfoGathererAgent.ts +339 -0
  153. package/src/prefabs/ReceptionistAgent.ts +366 -0
  154. package/src/prefabs/SurveyAgent.ts +646 -0
  155. package/src/prefabs/index.ts +27 -0
  156. package/src/relay/Action.ts +314 -0
  157. package/src/relay/Call.ts +854 -0
  158. package/src/relay/Deferred.ts +71 -0
  159. package/src/relay/Message.ts +148 -0
  160. package/src/relay/RelayClient.ts +1036 -0
  161. package/src/relay/RelayError.ts +14 -0
  162. package/src/relay/RelayEvent.ts +892 -0
  163. package/src/relay/constants.ts +132 -0
  164. package/src/relay/index.ts +123 -0
  165. package/src/relay/types.ts +149 -0
  166. package/src/rest/HttpClient.ts +103 -0
  167. package/src/rest/RestError.ts +31 -0
  168. package/src/rest/base/BaseResource.ts +19 -0
  169. package/src/rest/base/CrudResource.ts +51 -0
  170. package/src/rest/base/CrudWithAddresses.ts +26 -0
  171. package/src/rest/index.ts +200 -0
  172. package/src/rest/namespaces/addresses.ts +32 -0
  173. package/src/rest/namespaces/calling.ts +187 -0
  174. package/src/rest/namespaces/chat.ts +20 -0
  175. package/src/rest/namespaces/compat.ts +352 -0
  176. package/src/rest/namespaces/datasphere.ts +45 -0
  177. package/src/rest/namespaces/fabric.ts +222 -0
  178. package/src/rest/namespaces/imported-numbers.ts +20 -0
  179. package/src/rest/namespaces/logs.ts +84 -0
  180. package/src/rest/namespaces/lookup.ts +21 -0
  181. package/src/rest/namespaces/mfa.ts +30 -0
  182. package/src/rest/namespaces/number-groups.ts +38 -0
  183. package/src/rest/namespaces/phone-numbers.ts +44 -0
  184. package/src/rest/namespaces/project.ts +36 -0
  185. package/src/rest/namespaces/pubsub.ts +20 -0
  186. package/src/rest/namespaces/queues.ts +33 -0
  187. package/src/rest/namespaces/recordings.ts +28 -0
  188. package/src/rest/namespaces/registry.ts +101 -0
  189. package/src/rest/namespaces/short-codes.ts +28 -0
  190. package/src/rest/namespaces/sip-profile.ts +25 -0
  191. package/src/rest/namespaces/verified-callers.ts +27 -0
  192. package/src/rest/namespaces/video.ts +165 -0
  193. package/src/rest/pagination.ts +84 -0
  194. package/src/rest/types.ts +61 -0
  195. package/src/schema.json +12250 -0
  196. package/src/skills/SkillBase.ts +547 -0
  197. package/src/skills/SkillManager.ts +415 -0
  198. package/src/skills/SkillRegistry.ts +367 -0
  199. package/src/skills/builtin/api_ninjas_trivia.ts +280 -0
  200. package/src/skills/builtin/ask_claude.ts +241 -0
  201. package/src/skills/builtin/claude_skills.ts +978 -0
  202. package/src/skills/builtin/custom_skills.ts +312 -0
  203. package/src/skills/builtin/datasphere.ts +438 -0
  204. package/src/skills/builtin/datasphere_serverless.ts +339 -0
  205. package/src/skills/builtin/datetime.ts +130 -0
  206. package/src/skills/builtin/google_maps.ts +651 -0
  207. package/src/skills/builtin/index.ts +88 -0
  208. package/src/skills/builtin/info_gatherer.ts +405 -0
  209. package/src/skills/builtin/joke.ts +191 -0
  210. package/src/skills/builtin/math.ts +142 -0
  211. package/src/skills/builtin/mcp_gateway.ts +677 -0
  212. package/src/skills/builtin/native_vector_search.ts +901 -0
  213. package/src/skills/builtin/play_background_file.ts +358 -0
  214. package/src/skills/builtin/spider.ts +869 -0
  215. package/src/skills/builtin/swml_transfer.ts +635 -0
  216. package/src/skills/builtin/weather_api.ts +248 -0
  217. package/src/skills/builtin/web_search.ts +1074 -0
  218. package/src/skills/builtin/wikipedia_search.ts +299 -0
  219. package/src/skills/index.ts +12 -0
  220. package/src/types.ts +132 -0
  221. package/tests/AgentBase.test.ts +1093 -0
  222. package/tests/AgentServer.test.ts +126 -0
  223. package/tests/AuthHandler.test.ts +133 -0
  224. package/tests/ConfigLoader.test.ts +151 -0
  225. package/tests/ContextBuilder.test.ts +108 -0
  226. package/tests/DataMap.test.ts +367 -0
  227. package/tests/EphemeralSkillCopy.test.ts +107 -0
  228. package/tests/FunctionResult.test.ts +298 -0
  229. package/tests/GracefulShutdown.test.ts +46 -0
  230. package/tests/Logger.test.ts +420 -0
  231. package/tests/PomBuilder.test.ts +228 -0
  232. package/tests/ProxyDetection.test.ts +228 -0
  233. package/tests/SWMLService.test.ts +320 -0
  234. package/tests/SWMLVerbHandler.test.ts +523 -0
  235. package/tests/SchemaUtils.test.ts +106 -0
  236. package/tests/SchemaUtils.verb.test.ts +206 -0
  237. package/tests/SecurityUtils.test.ts +192 -0
  238. package/tests/ServerlessAdapter.test.ts +222 -0
  239. package/tests/SessionManager.test.ts +257 -0
  240. package/tests/SkillBaseFeatures.test.ts +130 -0
  241. package/tests/SkillMultiInstance.test.ts +102 -0
  242. package/tests/SkillParameterSchemas.test.ts +134 -0
  243. package/tests/SkillRegistrySchema.test.ts +84 -0
  244. package/tests/SslConfig.test.ts +97 -0
  245. package/tests/SwmlBuilder.test.ts +400 -0
  246. package/tests/TypeInference.test.ts +165 -0
  247. package/tests/VerbHandlerRegistry.test.ts +128 -0
  248. package/tests/cli/mock-data.test.ts +113 -0
  249. package/tests/cli/swaig-test.test.ts +108 -0
  250. package/tests/examples.test.ts +194 -0
  251. package/tests/fixtures/scrape-parity/README.md +53 -0
  252. package/tests/fixtures/scrape-parity/expected/01_simple_article.json +5 -0
  253. package/tests/fixtures/scrape-parity/expected/02_class_based_noise.json +5 -0
  254. package/tests/fixtures/scrape-parity/expected/03_nested_containers.json +5 -0
  255. package/tests/fixtures/scrape-parity/expected/04_bare_body.json +5 -0
  256. package/tests/fixtures/scrape-parity/expected/05_malformed.json +5 -0
  257. package/tests/fixtures/scrape-parity/expected/06_multi_class.json +5 -0
  258. package/tests/fixtures/scrape-parity/expected/07_anchor_pattern.json +5 -0
  259. package/tests/fixtures/scrape-parity/expected/08_inline_script.json +5 -0
  260. package/tests/fixtures/scrape-parity/expected/09_whitespace_nbsp.json +5 -0
  261. package/tests/fixtures/scrape-parity/expected/10_entity_unicode.json +5 -0
  262. package/tests/fixtures/scrape-parity/expected/11_cdata_comments.json +5 -0
  263. package/tests/fixtures/scrape-parity/expected/12_deep_nested_removals.json +5 -0
  264. package/tests/fixtures/scrape-parity/html/01_simple_article.html +18 -0
  265. package/tests/fixtures/scrape-parity/html/02_class_based_noise.html +18 -0
  266. package/tests/fixtures/scrape-parity/html/03_nested_containers.html +13 -0
  267. package/tests/fixtures/scrape-parity/html/04_bare_body.html +9 -0
  268. package/tests/fixtures/scrape-parity/html/05_malformed.html +11 -0
  269. package/tests/fixtures/scrape-parity/html/06_multi_class.html +18 -0
  270. package/tests/fixtures/scrape-parity/html/07_anchor_pattern.html +12 -0
  271. package/tests/fixtures/scrape-parity/html/08_inline_script.html +17 -0
  272. package/tests/fixtures/scrape-parity/html/09_whitespace_nbsp.html +21 -0
  273. package/tests/fixtures/scrape-parity/html/10_entity_unicode.html +10 -0
  274. package/tests/fixtures/scrape-parity/html/11_cdata_comments.html +13 -0
  275. package/tests/fixtures/scrape-parity/html/12_deep_nested_removals.html +24 -0
  276. package/tests/fixtures/scrape-parity/python_extract.py +109 -0
  277. package/tests/livewire/livewire.test.ts +780 -0
  278. package/tests/mcp.test.ts +221 -0
  279. package/tests/prefabs/prefabs.test.ts +1093 -0
  280. package/tests/relay/Action.test.ts +392 -0
  281. package/tests/relay/Call.test.ts +535 -0
  282. package/tests/relay/Deferred.test.ts +71 -0
  283. package/tests/relay/Message.test.ts +164 -0
  284. package/tests/relay/RelayClient.test.ts +779 -0
  285. package/tests/relay/RelayEvent.test.ts +463 -0
  286. package/tests/relay/helpers.ts +79 -0
  287. package/tests/rest/HttpClient.test.ts +274 -0
  288. package/tests/rest/RestError.test.ts +53 -0
  289. package/tests/rest/calling.test.ts +156 -0
  290. package/tests/rest/client.test.ts +197 -0
  291. package/tests/rest/compat.test.ts +335 -0
  292. package/tests/rest/datasphere.test.ts +71 -0
  293. package/tests/rest/fabric.test.ts +186 -0
  294. package/tests/rest/helpers.ts +95 -0
  295. package/tests/rest/logs.test.ts +68 -0
  296. package/tests/rest/pagination.test.ts +133 -0
  297. package/tests/rest/registry.test.ts +94 -0
  298. package/tests/rest/relay-rest.test.ts +290 -0
  299. package/tests/rest/video.test.ts +173 -0
  300. package/tests/setup.ts +6 -0
  301. package/tests/skills/SkillBase.test.ts +108 -0
  302. package/tests/skills/SkillManager.test.ts +268 -0
  303. package/tests/skills/SkillRegistry.test.ts +176 -0
  304. package/tests/skills/builtin.test.ts +1472 -0
  305. package/tests/skills/claude-skills.test.ts +54 -0
  306. package/tests/skills/custom-skills.test.ts +53 -0
  307. package/tests/skills/datasphere-serverless.test.ts +70 -0
  308. package/tests/skills/datasphere.test.ts +107 -0
  309. package/tests/skills/datetime.test.ts +101 -0
  310. package/tests/skills/google-maps.test.ts +78 -0
  311. package/tests/skills/info-gatherer.test.ts +411 -0
  312. package/tests/skills/joke.test.ts +81 -0
  313. package/tests/skills/math.test.ts +68 -0
  314. package/tests/skills/mcp-gateway.test.ts +101 -0
  315. package/tests/skills/play-background.test.ts +54 -0
  316. package/tests/skills/spider.test.ts +108 -0
  317. package/tests/skills/swml-transfer.test.ts +178 -0
  318. package/tests/skills/trivia.test.ts +54 -0
  319. package/tests/skills/vector-search.test.ts +135 -0
  320. package/tests/skills/weather-api.test.ts +77 -0
  321. package/tests/skills/web-search-parity.test.ts +59 -0
  322. package/tests/skills/web-search.test.ts +103 -0
  323. package/tests/skills/wikipedia.test.ts +63 -0
  324. package/tsconfig.json +18 -0
  325. package/vitest.config.ts +9 -0
@@ -0,0 +1,1074 @@
1
+ /**
2
+ * Web Search Skill - Searches the web using the Google Custom Search API.
3
+ *
4
+ * Tier 3 built-in skill: requires `GOOGLE_SEARCH_API_KEY` and
5
+ * `GOOGLE_SEARCH_ENGINE_ID` (legacy alias: `GOOGLE_SEARCH_CX`) environment
6
+ * variables, or the equivalent config params. Uses the Google Custom Search
7
+ * JSON API to perform web searches and return formatted results.
8
+ */
9
+
10
+ import { SkillBase } from '../SkillBase.js';
11
+ import type {
12
+ SkillToolDefinition,
13
+ SkillPromptSection,
14
+ SkillConfig,
15
+ ParameterSchemaEntry,
16
+ } from '../SkillBase.js';
17
+ import { FunctionResult } from '../../FunctionResult.js';
18
+ import { MAX_SKILL_INPUT_LENGTH, validateUrl } from '../../SecurityUtils.js';
19
+ import { getLogger } from '../../Logger.js';
20
+ import * as cheerio from 'cheerio';
21
+
22
+ const log = getLogger('WebSearchSkill');
23
+
24
+ const DEFAULT_NO_RESULTS_MESSAGE =
25
+ "I couldn't find quality results for '{query}'. " +
26
+ 'The search returned only low-quality or inaccessible pages. ' +
27
+ 'Try rephrasing your search or asking about a different topic.';
28
+
29
+ /** Selector priority — Python skill.py:220-228. */
30
+ const CONTENT_CANDIDATES = [
31
+ 'article',
32
+ 'main',
33
+ '[role="main"]',
34
+ '.content',
35
+ '#content',
36
+ '.post',
37
+ '.entry-content',
38
+ '.article-body',
39
+ '.story-body',
40
+ '.markdown-body',
41
+ '.wiki-body',
42
+ '.documentation',
43
+ ] as const;
44
+
45
+ /** Structural tags removed from the picked subtree — Python skill.py:239. */
46
+ const UNWANTED_TAGS = [
47
+ 'script',
48
+ 'style',
49
+ 'nav',
50
+ 'header',
51
+ 'footer',
52
+ 'aside',
53
+ 'noscript',
54
+ 'iframe',
55
+ ] as const;
56
+
57
+ /** Class/id name patterns removed from the picked subtree — Python skill.py:245-257. */
58
+ const UNWANTED_PATTERNS = [
59
+ 'sidebar',
60
+ 'navigation',
61
+ 'menu',
62
+ 'advertisement',
63
+ 'ads',
64
+ 'banner',
65
+ 'popup',
66
+ 'modal',
67
+ 'cookie',
68
+ 'gdpr',
69
+ 'subscribe',
70
+ 'newsletter',
71
+ 'comments',
72
+ 'related',
73
+ 'share',
74
+ 'social',
75
+ ] as const;
76
+
77
+ /**
78
+ * Run the HTML → plain-text extraction pipeline that mirrors Python's
79
+ * `GoogleSearchScraper.extract_html_content` (skill.py:204-282).
80
+ *
81
+ * Pipeline:
82
+ * 1. Pre-process `<![CDATA[…]]>` sections to plain text (Python html.parser
83
+ * keeps CDATA content; cheerio/htmlparser2 drops it).
84
+ * 2. Walk {@link CONTENT_CANDIDATES} in order, picking the first match.
85
+ * Fall back to `<body>` or the raw document.
86
+ * 3. Clone the picked subtree so tag/pattern removal only affects it —
87
+ * analogous to Python's `content_soup = BeautifulSoup(str(main_content))`.
88
+ * The selector-first-then-filter order is load-bearing: if the real
89
+ * content is wrapped in a sidebar-pattern div, Python still finds it
90
+ * because the selector runs before removal.
91
+ * 4. Remove {@link UNWANTED_TAGS} and every element matching any of
92
+ * {@link UNWANTED_PATTERNS} on either `class` or `id` (case-insensitive).
93
+ * 5. Collapse whitespace and trim.
94
+ *
95
+ * Exported so `tests/skills/web-search-parity.test.ts` can verify byte-
96
+ * identical behavior against Python BeautifulSoup fixtures; not re-exported
97
+ * from the skills barrel, so this stays internal to the skill module.
98
+ *
99
+ * @internal
100
+ */
101
+ export function extractTextFromHtml(html: string): string {
102
+ const normalized = html.replace(
103
+ /<!\[CDATA\[([\s\S]*?)\]\]>/g,
104
+ (_, content: string) => content,
105
+ );
106
+ const $ = cheerio.load(normalized);
107
+
108
+ let pickedHtml = '';
109
+ for (const sel of CONTENT_CANDIDATES) {
110
+ const picked = $(sel).first();
111
+ if (picked.length > 0) {
112
+ pickedHtml = $.html(picked);
113
+ break;
114
+ }
115
+ }
116
+ if (!pickedHtml) {
117
+ const bodyEl = $('body').first();
118
+ pickedHtml = bodyEl.length > 0 ? $.html(bodyEl) : normalized;
119
+ }
120
+
121
+ const $sub = cheerio.load(pickedHtml);
122
+
123
+ for (const tag of UNWANTED_TAGS) {
124
+ $sub(tag).remove();
125
+ }
126
+
127
+ for (const pattern of UNWANTED_PATTERNS) {
128
+ const regex = new RegExp(pattern, 'i');
129
+ $sub('[class]').each((_, el) => {
130
+ const cls = $sub(el).attr('class') ?? '';
131
+ if (regex.test(cls)) $sub(el).remove();
132
+ });
133
+ $sub('[id]').each((_, el) => {
134
+ const id = $sub(el).attr('id') ?? '';
135
+ if (regex.test(id)) $sub(el).remove();
136
+ });
137
+ }
138
+
139
+ return $sub.root().text().replace(/\s+/g, ' ').trim();
140
+ }
141
+
142
+ /** A single search result item from the Google Custom Search API. */
143
+ interface GoogleSearchItem {
144
+ /** Title of the search result. */
145
+ title: string;
146
+ /** URL of the search result. */
147
+ link: string;
148
+ /** Brief text snippet from the page. */
149
+ snippet: string;
150
+ /** Display-friendly domain name. */
151
+ displayLink?: string;
152
+ /** Structured page metadata. */
153
+ pagemap?: Record<string, unknown>;
154
+ }
155
+
156
+ /** Response shape from the Google Custom Search JSON API. */
157
+ interface GoogleSearchResponse {
158
+ /** Array of search result items. */
159
+ items?: GoogleSearchItem[];
160
+ /** Metadata about the search execution. */
161
+ searchInformation?: {
162
+ totalResults: string;
163
+ searchTime: number;
164
+ formattedTotalResults: string;
165
+ formattedSearchTime: string;
166
+ };
167
+ /** Error details if the request failed. */
168
+ error?: {
169
+ code: number;
170
+ message: string;
171
+ errors: Array<{ message: string; domain: string; reason: string }>;
172
+ };
173
+ }
174
+
175
+ /**
176
+ * Searches the web using the Google Custom Search JSON API.
177
+ *
178
+ * Tier 3 built-in skill. Credentials can be supplied via the `api_key` and
179
+ * `search_engine_id` params or `GOOGLE_SEARCH_API_KEY` /
180
+ * `GOOGLE_SEARCH_ENGINE_ID` (legacy: `GOOGLE_SEARCH_CX`) environment variables.
181
+ *
182
+ * The handler mirrors Python's `search_and_scrape_best` pipeline: fetches
183
+ * `oversample_factor × num_results` candidates from Google, scrapes each
184
+ * result page (SSRF-guarded, cheerio-based text extraction), scores for
185
+ * quality (length + query relevance + boilerplate penalty), deduplicates by
186
+ * domain, and returns the top `num_results` above `min_quality_score` with
187
+ * full page content. If every scrape fails or falls below the threshold the
188
+ * handler falls back to raw API snippets so the agent still has something
189
+ * to say.
190
+ *
191
+ * Supported config: `tool_name`, `num_results`, `no_results_message`,
192
+ * `safe_search`, `delay`, `max_content_length`, `oversample_factor`,
193
+ * `min_quality_score`.
194
+ */
195
+ export class WebSearchSkill extends SkillBase {
196
+ // Python ground truth: skills/web_search/skill.py:559-567
197
+ // REQUIRED_PACKAGES = ["bs4", "requests"] in Python; TS uses cheerio + fetch so [].
198
+ static override SKILL_NAME = 'web_search';
199
+ static override SKILL_DESCRIPTION =
200
+ 'Search the web for information using Google Custom Search API';
201
+ static override SKILL_VERSION = '2.0.0';
202
+ static override REQUIRED_PACKAGES: readonly string[] = [];
203
+ static override REQUIRED_ENV_VARS: readonly string[] = [];
204
+ static override SUPPORTS_MULTIPLE_INSTANCES = true;
205
+
206
+ static override getParameterSchema(): Record<string, ParameterSchemaEntry> {
207
+ return {
208
+ ...super.getParameterSchema(),
209
+ api_key: {
210
+ type: 'string',
211
+ description: 'Google Custom Search API key',
212
+ required: true,
213
+ hidden: true,
214
+ env_var: 'GOOGLE_SEARCH_API_KEY',
215
+ },
216
+ search_engine_id: {
217
+ type: 'string',
218
+ description: 'Google Custom Search Engine ID',
219
+ required: true,
220
+ hidden: true,
221
+ env_var: 'GOOGLE_SEARCH_ENGINE_ID',
222
+ },
223
+ tool_name: {
224
+ type: 'string',
225
+ description: 'Custom tool name for this Web Search instance.',
226
+ required: false,
227
+ },
228
+ num_results: {
229
+ type: 'integer',
230
+ description: 'Number of high-quality results to return',
231
+ default: 3,
232
+ required: false,
233
+ min: 1,
234
+ max: 10,
235
+ },
236
+ delay: {
237
+ type: 'number',
238
+ description: 'Delay between scraping pages in seconds',
239
+ default: 0.5,
240
+ required: false,
241
+ min: 0,
242
+ },
243
+ max_content_length: {
244
+ type: 'integer',
245
+ description: 'Maximum total response size in characters',
246
+ default: 32768,
247
+ required: false,
248
+ min: 1000,
249
+ },
250
+ oversample_factor: {
251
+ type: 'number',
252
+ description:
253
+ 'How many extra results to fetch for quality filtering (e.g., 2.5 = fetch 2.5x requested)',
254
+ default: 2.5,
255
+ required: false,
256
+ min: 1.0,
257
+ max: 3.5,
258
+ },
259
+ min_quality_score: {
260
+ type: 'number',
261
+ description: 'Minimum quality score (0-1) for including a result',
262
+ default: 0.3,
263
+ required: false,
264
+ min: 0,
265
+ max: 1,
266
+ },
267
+ no_results_message: {
268
+ type: 'string',
269
+ description:
270
+ 'Message to show when no quality results are found. Use {query} as placeholder.',
271
+ default: DEFAULT_NO_RESULTS_MESSAGE,
272
+ required: false,
273
+ },
274
+ safe_search: {
275
+ type: 'string',
276
+ description: 'Safe search level.',
277
+ default: 'medium',
278
+ enum: ['off', 'medium', 'high'],
279
+ },
280
+ };
281
+ }
282
+
283
+ /**
284
+ * @returns Manifest declaring Google Search credentials as required env vars.
285
+ * Reports `GOOGLE_SEARCH_ENGINE_ID` as the canonical name; `GOOGLE_SEARCH_CX`
286
+ * is still accepted as a legacy fallback at runtime.
287
+ */
288
+ /**
289
+ * Validate required credentials before the skill becomes active.
290
+ *
291
+ * Mirrors Python's `setup()` (skill.py:559-600) which checks `api_key` and
292
+ * `search_engine_id` and returns `False` (logging an error) if either is
293
+ * absent. In the TS SDK credentials may also arrive via environment variables
294
+ * (`GOOGLE_SEARCH_API_KEY` / `GOOGLE_SEARCH_ENGINE_ID` or the legacy alias
295
+ * `GOOGLE_SEARCH_CX`), so both config params and env vars are checked.
296
+ * @returns `true` if all required credentials are present, `false` otherwise.
297
+ */
298
+ override async setup(): Promise<boolean> {
299
+ const apiKey =
300
+ this.getConfig<string | undefined>('api_key', undefined) ??
301
+ process.env['GOOGLE_SEARCH_API_KEY'];
302
+ const searchEngineId =
303
+ this.getConfig<string | undefined>('search_engine_id', undefined) ??
304
+ process.env['GOOGLE_SEARCH_ENGINE_ID'] ??
305
+ process.env['GOOGLE_SEARCH_CX'];
306
+
307
+ const missing: string[] = [];
308
+ if (!apiKey) missing.push('api_key / GOOGLE_SEARCH_API_KEY');
309
+ if (!searchEngineId) missing.push('search_engine_id / GOOGLE_SEARCH_ENGINE_ID');
310
+
311
+ if (missing.length > 0) {
312
+ log.error('web_search: missing required parameters', { missing });
313
+ return false;
314
+ }
315
+ return true;
316
+ }
317
+
318
+ /**
319
+ * Instance key for the SkillManager. Includes the configured
320
+ * `search_engine_id` (or `"default"`) and `tool_name` (or `"web_search"`)
321
+ * to match Python's `"{SKILL_NAME}_{search_engine_id}_{tool_name}"` scheme.
322
+ */
323
+ override getInstanceKey(): string {
324
+ const searchEngineId = this.getConfig<string>('search_engine_id', '') || 'default';
325
+ const toolName = this.getConfig<string>('tool_name', this.skillName);
326
+ return `${this.skillName}_${searchEngineId}_${toolName}`;
327
+ }
328
+
329
+ /** Global data injected into the agent's SWML context (mirrors Python). */
330
+ override getGlobalData(): Record<string, unknown> {
331
+ return {
332
+ web_search_enabled: true,
333
+ search_provider: 'Google Custom Search',
334
+ quality_filtering: true,
335
+ };
336
+ }
337
+
338
+ /** Resolve the tool name (defaults to `web_search`, matches Python default). */
339
+ private getToolName(): string {
340
+ return this.getConfig<string>('tool_name', 'web_search');
341
+ }
342
+
343
+ /**
344
+ * @returns A single tool (named via `tool_name`) that performs a Google
345
+ * Custom Search and returns formatted results.
346
+ */
347
+ getTools(): SkillToolDefinition[] {
348
+ const configNumResults = this.getConfig<number>('num_results', 3);
349
+ const safeSearch = this.getConfig<string>('safe_search', 'medium');
350
+ const oversampleFactor = this.getConfig<number>('oversample_factor', 2.5);
351
+ const minQualityScore = this.getConfig<number>('min_quality_score', 0.3);
352
+ const maxContentLength = this.getConfig<number>('max_content_length', 32768);
353
+ const delaySeconds = this.getConfig<number>('delay', 0.5);
354
+ const noResultsMessage = this.getConfig<string>(
355
+ 'no_results_message',
356
+ DEFAULT_NO_RESULTS_MESSAGE,
357
+ );
358
+ const toolName = this.getToolName();
359
+
360
+ return [
361
+ {
362
+ name: toolName,
363
+ description:
364
+ 'Search the web for high-quality information, automatically filtering low-quality results',
365
+ parameters: {
366
+ query: {
367
+ type: 'string',
368
+ description:
369
+ "The search query - what you want to find information about",
370
+ },
371
+ },
372
+ required: ['query'],
373
+ handler: async (args: Record<string, unknown>, _rawData: Record<string, unknown>) => {
374
+ const rawQuery = args['query'];
375
+
376
+ if (
377
+ !rawQuery ||
378
+ typeof rawQuery !== 'string' ||
379
+ rawQuery.trim().length === 0
380
+ ) {
381
+ return new FunctionResult(
382
+ 'Please provide a search query. What would you like me to search for?',
383
+ );
384
+ }
385
+
386
+ const query = rawQuery.trim();
387
+
388
+ if (query.length > MAX_SKILL_INPUT_LENGTH) {
389
+ return new FunctionResult('Search query is too long.');
390
+ }
391
+
392
+ const apiKey =
393
+ this.getConfig<string | undefined>('api_key', undefined) ??
394
+ process.env['GOOGLE_SEARCH_API_KEY'];
395
+ const searchEngineId =
396
+ this.getConfig<string | undefined>('search_engine_id', undefined) ??
397
+ process.env['GOOGLE_SEARCH_ENGINE_ID'] ??
398
+ process.env['GOOGLE_SEARCH_CX'];
399
+
400
+ if (!apiKey || !searchEngineId) {
401
+ return new FunctionResult(
402
+ 'Service is not configured. Please contact your administrator.',
403
+ );
404
+ }
405
+
406
+ const count = Math.max(1, Math.min(10, configNumResults));
407
+ // Python skill.py:433 — `fetch_count = min(10, int(num_results * oversample_factor))`.
408
+ // Python's `int()` truncates toward zero; for positive values that's
409
+ // equivalent to Math.floor. Do NOT add a `max(count, …)` floor —
410
+ // Python's fetch_count can be less than num_results (e.g. factor=0.5).
411
+ const fetchCount = Math.min(
412
+ 10,
413
+ Math.floor(count * oversampleFactor),
414
+ );
415
+
416
+ try {
417
+ const encodedQuery = encodeURIComponent(query);
418
+ const safeParam = safeSearch !== 'off' ? `&safe=${safeSearch}` : '';
419
+ const url =
420
+ `https://www.googleapis.com/customsearch/v1` +
421
+ // Python uses timeout=15 on the API request (skill.py:220).
422
+ `?key=${apiKey}&cx=${searchEngineId}&q=${encodedQuery}&num=${fetchCount}${safeParam}`;
423
+
424
+ const controller = new AbortController();
425
+ const timeout = setTimeout(() => controller.abort(), 15_000);
426
+ let response: Response;
427
+ try {
428
+ response = await fetch(url, { signal: controller.signal });
429
+ } finally {
430
+ clearTimeout(timeout);
431
+ }
432
+ const data = (await response.json()) as GoogleSearchResponse;
433
+
434
+ if (data.error) {
435
+ log.error('web_search_api_error', { code: data.error.code });
436
+ return new FunctionResult(
437
+ 'The web search service encountered an error. Please try again later.',
438
+ );
439
+ }
440
+
441
+ if (!data.items || data.items.length === 0) {
442
+ return new FunctionResult(
443
+ WebSearchSkill._formatNoResultsMessage(noResultsMessage, query),
444
+ );
445
+ }
446
+
447
+ // Scrape each result page, score for quality, dedupe by domain,
448
+ // return the top `count` above the min threshold. Port of
449
+ // Python's GoogleSearchScraper.search_and_scrape_best
450
+ // (skill.py:416-526).
451
+ //
452
+ // Python scrapes each URL up to `self.max_content_length` chars
453
+ // (skill.py:274-277 default), NOT the per-result budget. Metrics
454
+ // are computed on the scraped text, then output-time truncation
455
+ // applies the per-result budget. Mirror that.
456
+ type Candidate = {
457
+ item: GoogleSearchItem;
458
+ text: string;
459
+ score: number;
460
+ host: string;
461
+ text_length: number;
462
+ sentence_count: number;
463
+ query_relevance: number;
464
+ query_words_found: string;
465
+ domain: string;
466
+ };
467
+ const allCandidates: Candidate[] = [];
468
+
469
+ for (let idx = 0; idx < data.items.length; idx++) {
470
+ const item = data.items[idx];
471
+ try {
472
+ // SSRF / URL sanity — Python has no explicit pre-check; the
473
+ // scraper's inner `validate_url` handles it. We pre-parse to
474
+ // derive `host` for the domain-diverse selection pass.
475
+ const host = new URL(item.link).host;
476
+ const result = await this._scrapeUrl(
477
+ item.link,
478
+ maxContentLength,
479
+ 10_000,
480
+ query,
481
+ );
482
+ if (result !== null && result.score >= minQualityScore) {
483
+ allCandidates.push({
484
+ item,
485
+ text: result.text,
486
+ score: result.score,
487
+ host,
488
+ text_length: result.text_length,
489
+ sentence_count: result.sentence_count,
490
+ query_relevance: result.query_relevance,
491
+ query_words_found: result.query_words_found,
492
+ domain: result.domain,
493
+ });
494
+ }
495
+ } catch {
496
+ // URL parse failure — skip this result but still apply the
497
+ // inter-request delay below so cadence matches Python.
498
+ }
499
+
500
+ // Python skill.py:461-463 sleeps `delay` seconds between every
501
+ // iteration regardless of success/failure. Skip on the last
502
+ // iteration because there is nothing left to fetch.
503
+ if (delaySeconds > 0 && idx < data.items.length - 1) {
504
+ await new Promise((r) => setTimeout(r, delaySeconds * 1000));
505
+ }
506
+ }
507
+
508
+ // Two-pass selection: sort by score desc, then (1) take the
509
+ // highest-quality result per unique host, (2) fill remaining
510
+ // slots from the leftovers. Mirrors Python skill.py:~495-540.
511
+ allCandidates.sort((a, b) => b.score - a.score);
512
+ const chosen: Candidate[] = [];
513
+ const chosenHosts = new Set<string>();
514
+ for (const cand of allCandidates) {
515
+ if (chosen.length >= count) break;
516
+ if (!chosenHosts.has(cand.host)) {
517
+ chosen.push(cand);
518
+ chosenHosts.add(cand.host);
519
+ }
520
+ }
521
+ if (chosen.length < count) {
522
+ for (const cand of allCandidates) {
523
+ if (chosen.length >= count) break;
524
+ if (!chosen.includes(cand)) chosen.push(cand);
525
+ }
526
+ }
527
+ const scored = chosen;
528
+
529
+ if (allCandidates.length === 0 || scored.length === 0) {
530
+ // Python skill.py:465-466 / 488-489 return a plain "no quality
531
+ // results" string, which the outer handler (skill.py:640-642)
532
+ // detects and replaces with the configured `no_results_message`.
533
+ // Skip the middleman and return the configured message directly.
534
+ return new FunctionResult(
535
+ WebSearchSkill._formatNoResultsMessage(noResultsMessage, query),
536
+ );
537
+ }
538
+
539
+ // Per-result content budget — Python skill.py:491-495 divides
540
+ // by `len(best_results)` (the chosen count), not `num_results`,
541
+ // so under-filled result sets get more room per source.
542
+ const overheadPerResult = 400;
543
+ const availableForContent =
544
+ maxContentLength - scored.length * overheadPerResult;
545
+ const perResultBudget = Math.max(
546
+ 2000,
547
+ Math.floor(availableForContent / scored.length),
548
+ );
549
+
550
+ // Ranked-by-score output with full page content.
551
+ // Python skill.py:499-524 format — header lines + per-result
552
+ // block with Title / URL / Source / Snippet / Content Stats /
553
+ // Query Relevance / Content + 50-char separator.
554
+ const separator = '='.repeat(50);
555
+ const innerLines: string[] = [
556
+ `Found ${allCandidates.length} results meeting quality threshold from ${data.items.length} searched.`,
557
+ `Showing top ${scored.length} from diverse sources:\n`,
558
+ ];
559
+ for (let i = 0; i < scored.length; i++) {
560
+ const cand = scored[i];
561
+ let block = `=== RESULT ${i + 1} (Quality: ${cand.score.toFixed(2)}) ===\n`;
562
+ block += `Title: ${cand.item.title}\n`;
563
+ block += `URL: ${cand.item.link}\n`;
564
+ block += `Source: ${cand.domain}\n`;
565
+ // Python skill.py:507 always prints `Snippet:` (empty when the
566
+ // Google result has no snippet). Mirror that for a stable layout.
567
+ const rawSnippet = cand.item.snippet ?? '';
568
+ block += `Snippet: ${rawSnippet.replace(/\n/g, ' ').trim()}\n`;
569
+ block += `Content Stats: ${cand.text_length} chars, ${cand.sentence_count} sentences\n`;
570
+ block += `Query Relevance: ${cand.query_relevance.toFixed(2)} (keywords: ${cand.query_words_found})\n`;
571
+ block += `Content:\n`;
572
+ let content = cand.text;
573
+ if (content.length > perResultBudget) {
574
+ content = content.slice(0, perResultBudget) + '...';
575
+ }
576
+ block += content;
577
+ // Python skill.py:523 — `f"\n{'='*50}\n\n"` (two trailing newlines).
578
+ block += `\n${separator}\n\n`;
579
+ innerLines.push(block);
580
+ }
581
+ const innerOutput = innerLines.join('\n');
582
+ // Python's outer `_web_search_handler` (skill.py:644) wraps the
583
+ // inner formatter output with a `"Quality web search results for
584
+ // '<query>':\n\n"` preamble.
585
+ return new FunctionResult(
586
+ `Quality web search results for '${query}':\n\n${innerOutput}`,
587
+ );
588
+ } catch (err) {
589
+ log.error('web_search_failed', {
590
+ error: err instanceof Error ? err.message : String(err),
591
+ });
592
+ return new FunctionResult(
593
+ 'Sorry, I encountered an error while searching. Please try again later.',
594
+ );
595
+ }
596
+ },
597
+ },
598
+ ];
599
+ }
600
+
601
+ /** Apply the `{query}` template to the no-results message. */
602
+ private static _formatNoResultsMessage(template: string, query: string): string {
603
+ return template.includes('{query}')
604
+ ? template.replace(/\{query\}/g, query)
605
+ : template;
606
+ }
607
+
608
+ /**
609
+ * Check whether the URL points at Reddit. Python parity: `is_reddit_url`
610
+ * (skill.py:66).
611
+ */
612
+ private static _isRedditUrl(url: string): boolean {
613
+ try {
614
+ const host = new URL(url).hostname.toLowerCase();
615
+ return host.includes('reddit.com') || host.includes('redd.it');
616
+ } catch {
617
+ return false;
618
+ }
619
+ }
620
+
621
+ /**
622
+ * Fetch a Reddit URL via the `.json` endpoint and build a structured summary
623
+ * of the post + top comments.
624
+ *
625
+ * Python parity: `extract_reddit_content` (skill.py:71-190). Matches the
626
+ * post-title/author/score/comments assembly and the top-20 → valid → top-5
627
+ * comment pipeline. Returns just the compiled text — Python's
628
+ * `search_and_scrape_best` unconditionally overwrites Reddit's
629
+ * engagement-score metrics with the 6-factor `_calculate_content_quality`
630
+ * (skill.py:447-448), so we skip computing the dead engagement score here
631
+ * and let the handler score the text via `_qualityMetrics`.
632
+ *
633
+ * Falls through to HTML extraction on JSON fetch failure or malformed
634
+ * payload, matching Python's `except Exception: fall back` behavior.
635
+ */
636
+ private async _extractRedditContent(
637
+ url: string,
638
+ contentLimit: number,
639
+ timeoutMs: number,
640
+ ): Promise<{ text: string } | null> {
641
+ try {
642
+ if (!(await validateUrl(url))) {
643
+ log.debug('web_search: reddit URL rejected by SSRF guard', { url });
644
+ return null;
645
+ }
646
+ const jsonUrl = url.endsWith('.json')
647
+ ? url
648
+ : `${url.replace(/\/+$/, '')}.json`;
649
+ const controller = new AbortController();
650
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
651
+ let response: Response;
652
+ try {
653
+ response = await fetch(jsonUrl, {
654
+ signal: controller.signal,
655
+ headers: { 'User-Agent': 'SignalWire-WebSearch/2.0' },
656
+ });
657
+ } finally {
658
+ clearTimeout(timer);
659
+ }
660
+ if (!response.ok) {
661
+ log.debug('web_search: reddit JSON non-200', {
662
+ url: jsonUrl,
663
+ status: response.status,
664
+ });
665
+ return null;
666
+ }
667
+ const data = (await response.json()) as unknown;
668
+ if (!Array.isArray(data) || data.length < 1) {
669
+ log.debug('web_search: reddit JSON structure invalid', { url });
670
+ return null;
671
+ }
672
+
673
+ const listing0 = data[0] as {
674
+ data?: { children?: Array<{ data?: Record<string, unknown> }> };
675
+ };
676
+ const postEntry = listing0?.data?.children?.[0]?.data;
677
+ if (!postEntry) {
678
+ log.debug('web_search: reddit post missing', { url });
679
+ return null;
680
+ }
681
+
682
+ const title = (postEntry['title'] as string | undefined) ?? 'No title';
683
+ const author = (postEntry['author'] as string | undefined) ?? 'unknown';
684
+ const postScore = (postEntry['score'] as number | undefined) ?? 0;
685
+ const numComments = (postEntry['num_comments'] as number | undefined) ?? 0;
686
+ const subreddit = (postEntry['subreddit'] as string | undefined) ?? '';
687
+ const selftext = ((postEntry['selftext'] as string | undefined) ?? '').trim();
688
+
689
+ const parts: string[] = [];
690
+ parts.push(`Reddit r/${subreddit} Discussion`);
691
+ parts.push(`\nPost: ${title}`);
692
+ parts.push(
693
+ `Author: ${author} | Score: ${postScore} | Comments: ${numComments}`,
694
+ );
695
+ if (selftext && selftext !== '[removed]' && selftext !== '[deleted]') {
696
+ parts.push(`\nOriginal Post:\n${selftext.slice(0, 1000)}`);
697
+ }
698
+
699
+ const validComments: Array<{ body: string; author: string; score: number }> = [];
700
+ if (data.length > 1) {
701
+ const listing1 = data[1] as {
702
+ data?: {
703
+ children?: Array<{ kind?: string; data?: Record<string, unknown> }>;
704
+ };
705
+ };
706
+ const comments = listing1?.data?.children ?? [];
707
+ for (const c of comments.slice(0, 20)) {
708
+ if (c.kind !== 't1' || !c.data) continue;
709
+ const body = ((c.data['body'] as string | undefined) ?? '').trim();
710
+ if (
711
+ !body ||
712
+ body === '[removed]' ||
713
+ body === '[deleted]' ||
714
+ body.length <= 50
715
+ ) {
716
+ continue;
717
+ }
718
+ validComments.push({
719
+ body,
720
+ author: (c.data['author'] as string | undefined) ?? 'unknown',
721
+ score: (c.data['score'] as number | undefined) ?? 0,
722
+ });
723
+ }
724
+ validComments.sort((a, b) => b.score - a.score);
725
+
726
+ if (validComments.length > 0) {
727
+ parts.push('\n--- Top Discussion ---');
728
+ validComments.slice(0, 5).forEach((c, i) => {
729
+ let commentText = c.body.slice(0, 500);
730
+ if (c.body.length > 500) commentText += '...';
731
+ parts.push(
732
+ `\nComment ${i + 1} (Score: ${c.score}, Author: ${c.author}):`,
733
+ );
734
+ parts.push(commentText);
735
+ });
736
+ }
737
+ }
738
+
739
+ let text = parts.join('\n');
740
+
741
+ if (text.length > contentLimit) {
742
+ text = text.slice(0, contentLimit);
743
+ }
744
+
745
+ return { text };
746
+ } catch (err) {
747
+ log.debug('web_search: reddit extraction failed', {
748
+ url,
749
+ error: err instanceof Error ? err.message : String(err),
750
+ });
751
+ return null;
752
+ }
753
+ }
754
+
755
+ /**
756
+ * Fetch a URL and extract clean text content, then score it with the
757
+ * 6-factor `_qualityMetrics`. Reddit URLs are routed to the JSON extractor
758
+ * first; the compiled Reddit text is scored with the same 6-factor formula
759
+ * to match Python's `search_and_scrape_best` overwrite behavior
760
+ * (skill.py:447-448).
761
+ *
762
+ * Python parity: `extract_text_from_url` (skill.py:192-202). Returns `null`
763
+ * on any failure (network, non-200, parse error, or SSRF rejection).
764
+ */
765
+ private async _scrapeUrl(
766
+ url: string,
767
+ contentLimit: number,
768
+ timeoutMs: number,
769
+ query: string,
770
+ ): Promise<
771
+ | {
772
+ text: string;
773
+ score: number;
774
+ text_length: number;
775
+ sentence_count: number;
776
+ query_relevance: number;
777
+ query_words_found: string;
778
+ domain: string;
779
+ }
780
+ | null
781
+ > {
782
+ if (WebSearchSkill._isRedditUrl(url)) {
783
+ const reddit = await this._extractRedditContent(url, contentLimit, timeoutMs);
784
+ if (reddit) {
785
+ const metrics = WebSearchSkill._qualityMetrics(reddit.text, url, query);
786
+ return { text: reddit.text, ...metrics };
787
+ }
788
+ // Python skill.py:189-190 falls through to HTML extraction if the
789
+ // Reddit JSON path fails. Mirror that.
790
+ }
791
+
792
+ // SSRF guard — Python's scraper calls validate_url() before every
793
+ // requests.get() (skill.py:79-80, 209-210).
794
+ if (!(await validateUrl(url))) {
795
+ log.debug('web_search: scrape URL rejected by SSRF guard', { url });
796
+ return null;
797
+ }
798
+ const controller = new AbortController();
799
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
800
+ try {
801
+ const response = await fetch(url, {
802
+ signal: controller.signal,
803
+ headers: {
804
+ 'User-Agent':
805
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
806
+ },
807
+ });
808
+ if (!response.ok) {
809
+ log.debug('web_search: scrape non-200', { url, status: response.status });
810
+ return null;
811
+ }
812
+ const body = await response.text();
813
+ let text = extractTextFromHtml(body);
814
+ if (text.length > contentLimit) {
815
+ text = text.slice(0, contentLimit);
816
+ }
817
+ if (!text) return null;
818
+ const metrics = WebSearchSkill._qualityMetrics(text, url, query);
819
+ return { text, ...metrics };
820
+ } catch (err) {
821
+ log.debug('web_search: scrape failed', {
822
+ url,
823
+ error: err instanceof Error ? err.message : String(err),
824
+ });
825
+ return null;
826
+ } finally {
827
+ clearTimeout(timer);
828
+ }
829
+ }
830
+
831
+ /**
832
+ * Six-factor content quality metrics (score + sub-metrics used in the
833
+ * per-result output). Combines content length, word diversity,
834
+ * boilerplate penalty, sentence structure, domain reputation, and query
835
+ * relevance.
836
+ *
837
+ * Python parity: `_calculate_content_quality` (skill.py:284-414). Preserves
838
+ * the weights (0.25/0.10/0.10/0.15/0.15/0.25), the 26-phrase boilerplate
839
+ * list, the quality/low-quality domain lists, and the phrase-match bonus
840
+ * on relevance. Also returns the same metric fields Python exposes
841
+ * (`text_length`, `sentence_count`, `query_relevance`, `query_words_found`,
842
+ * `domain`) so the handler can render the full Python output format.
843
+ */
844
+ private static _qualityMetrics(
845
+ text: string,
846
+ url: string,
847
+ query: string,
848
+ ): {
849
+ score: number;
850
+ text_length: number;
851
+ sentence_count: number;
852
+ query_relevance: number;
853
+ query_words_found: string;
854
+ domain: string;
855
+ } {
856
+ if (!text) {
857
+ return {
858
+ score: 0,
859
+ text_length: 0,
860
+ sentence_count: 0,
861
+ query_relevance: 0,
862
+ query_words_found: 'N/A',
863
+ domain: '',
864
+ };
865
+ }
866
+
867
+ // 1. Length score — tiered bands (Python skill.py:300-310).
868
+ //
869
+ // Python's `len(text)` counts codepoints; JS's `text.length` counts
870
+ // UTF-16 code units, so any character outside the BMP (emoji, astral
871
+ // plane letters) counts as 2 in JS and 1 in Python. For emoji-heavy
872
+ // content that would inflate the TS length score relative to Python.
873
+ // Count codepoints explicitly to match Python.
874
+ const textLength = [...text].length;
875
+ let lengthScore: number;
876
+ if (textLength < 500) {
877
+ lengthScore = 0;
878
+ } else if (textLength < 2000) {
879
+ lengthScore = ((textLength - 500) / 1500) * 0.5;
880
+ } else if (textLength <= 10000) {
881
+ lengthScore = 1.0;
882
+ } else {
883
+ lengthScore = Math.max(0.8, 1.0 - (textLength - 10000) / 20000);
884
+ }
885
+
886
+ // 2. Word diversity — unique / (total * 0.3). Python skill.py:313-321.
887
+ const words = text.toLowerCase().split(/\s+/).filter(Boolean);
888
+ const diversityScore =
889
+ words.length === 0
890
+ ? 0
891
+ : Math.min(1, new Set(words).size / (words.length * 0.3));
892
+
893
+ // 3. Boilerplate penalty — 26 phrases, -15% per phrase.
894
+ // Python skill.py:324-335.
895
+ const boilerplatePhrases = [
896
+ 'cookie',
897
+ 'privacy policy',
898
+ 'terms of service',
899
+ 'subscribe',
900
+ 'sign up',
901
+ 'log in',
902
+ 'advertisement',
903
+ 'sponsored',
904
+ 'copyright',
905
+ 'all rights reserved',
906
+ 'skip to',
907
+ 'navigation',
908
+ 'breadcrumb',
909
+ 'reddit inc',
910
+ 'google llc',
911
+ 'expand navigation',
912
+ 'members •',
913
+ 'archived post',
914
+ 'votes cannot be cast',
915
+ 'r/',
916
+ 'subreddit',
917
+ 'youtube',
918
+ 'facebook',
919
+ 'twitter',
920
+ 'instagram',
921
+ 'pinterest',
922
+ ];
923
+ const lowerText = text.toLowerCase();
924
+ const boilerplateCount = boilerplatePhrases.reduce(
925
+ (n, phrase) => (lowerText.includes(phrase) ? n + 1 : n),
926
+ 0,
927
+ );
928
+ const boilerplatePenalty = Math.max(0, 1 - boilerplateCount * 0.15);
929
+
930
+ // 4. Sentence score — count sentences with >30 chars; target 10.
931
+ // Python skill.py:339-342.
932
+ const sentences = text
933
+ .split(/[.!?]+/)
934
+ .filter((s) => s.trim().length > 30);
935
+ const sentenceScore = Math.min(1, sentences.length / 10);
936
+
937
+ // 5. Domain score — quality domains +1.5×, low-quality 0.1×, else 1.0.
938
+ // Python skill.py:344-365.
939
+ let domain = '';
940
+ try {
941
+ domain = new URL(url).hostname.toLowerCase();
942
+ } catch {
943
+ domain = '';
944
+ }
945
+ const qualityDomains = [
946
+ 'wikipedia.org',
947
+ 'starwars.fandom.com',
948
+ 'imdb.com',
949
+ 'screenrant.com',
950
+ 'denofgeek.com',
951
+ 'ign.com',
952
+ 'hollywoodreporter.com',
953
+ 'variety.com',
954
+ 'ew.com',
955
+ 'stackexchange.com',
956
+ 'stackoverflow.com',
957
+ 'github.com',
958
+ 'medium.com',
959
+ 'dev.to',
960
+ 'arxiv.org',
961
+ 'nature.com',
962
+ 'sciencedirect.com',
963
+ 'ieee.org',
964
+ ];
965
+ const lowQualityDomains = [
966
+ 'reddit.com',
967
+ 'youtube.com',
968
+ 'facebook.com',
969
+ 'twitter.com',
970
+ 'instagram.com',
971
+ 'pinterest.com',
972
+ 'tiktok.com',
973
+ 'x.com',
974
+ ];
975
+ let domainScore = 1.0;
976
+ if (qualityDomains.some((d) => domain.includes(d))) {
977
+ domainScore = 1.5;
978
+ } else if (lowQualityDomains.some((d) => domain.includes(d))) {
979
+ domainScore = 0.1;
980
+ }
981
+
982
+ // 6. Query relevance — word overlap + phrase bonus.
983
+ // Python skill.py:370-395.
984
+ const stopWords = new Set([
985
+ 'the',
986
+ 'a',
987
+ 'an',
988
+ 'and',
989
+ 'or',
990
+ 'but',
991
+ 'in',
992
+ 'on',
993
+ 'at',
994
+ 'to',
995
+ 'for',
996
+ 'of',
997
+ 'with',
998
+ 'by',
999
+ 'from',
1000
+ 'as',
1001
+ 'is',
1002
+ 'was',
1003
+ 'are',
1004
+ 'were',
1005
+ ]);
1006
+ const queryWords = query
1007
+ .toLowerCase()
1008
+ .split(/\s+/)
1009
+ .filter((w) => w.length > 2 && !stopWords.has(w));
1010
+ let relevanceScore: number;
1011
+ let wordsFound = 'N/A';
1012
+ if (queryWords.length === 0) {
1013
+ relevanceScore = 0.5;
1014
+ } else {
1015
+ const lowerContent = text.toLowerCase();
1016
+ let hits = 0;
1017
+ for (const w of queryWords) {
1018
+ if (lowerContent.includes(w)) hits++;
1019
+ }
1020
+ let phraseBonus = 0;
1021
+ for (let i = 0; i < queryWords.length - 1; i++) {
1022
+ const phrase = `${queryWords[i]} ${queryWords[i + 1]}`;
1023
+ if (lowerContent.includes(phrase)) phraseBonus += 0.2;
1024
+ }
1025
+ relevanceScore = Math.min(1, hits / queryWords.length + phraseBonus);
1026
+ // Python skill.py:392 — `"{words_found}/{len(query_words)}"` string.
1027
+ wordsFound = `${hits}/${queryWords.length}`;
1028
+ }
1029
+
1030
+ // Weighted combination (Python skill.py:397-405).
1031
+ const score =
1032
+ lengthScore * 0.25 +
1033
+ diversityScore * 0.1 +
1034
+ boilerplatePenalty * 0.1 +
1035
+ sentenceScore * 0.15 +
1036
+ domainScore * 0.15 +
1037
+ relevanceScore * 0.25;
1038
+
1039
+ return {
1040
+ score,
1041
+ text_length: textLength,
1042
+ sentence_count: sentences.length,
1043
+ query_relevance: relevanceScore,
1044
+ query_words_found: wordsFound,
1045
+ domain,
1046
+ };
1047
+ }
1048
+
1049
+ /** @returns Prompt section describing web search capabilities and usage guidance. */
1050
+ protected override _getPromptSections(): SkillPromptSection[] {
1051
+ const toolName = this.getToolName();
1052
+ return [
1053
+ {
1054
+ title: 'Web Search Capability (Quality Enhanced)',
1055
+ body: `You can search the internet for high-quality information using the ${toolName} tool.`,
1056
+ bullets: [
1057
+ `Use the ${toolName} tool when users ask for information you need to look up`,
1058
+ 'The search automatically filters out low-quality results like empty pages',
1059
+ 'Results are ranked by content quality, relevance, and domain reputation',
1060
+ 'Summarize the high-quality results in a clear, helpful way',
1061
+ ],
1062
+ },
1063
+ ];
1064
+ }
1065
+ }
1066
+
1067
+ /**
1068
+ * Factory function for creating WebSearchSkill instances.
1069
+ * @param config - Optional skill configuration.
1070
+ * @returns A new WebSearchSkill instance.
1071
+ */
1072
+ export function createSkill(config?: SkillConfig): WebSearchSkill {
1073
+ return new WebSearchSkill(config);
1074
+ }