@staticn0va/wigolo 0.3.1 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. package/README.md +57 -2
  2. package/SKILL.md +196 -0
  3. package/dist/agent/executor.d.ts +13 -0
  4. package/dist/agent/executor.d.ts.map +1 -0
  5. package/dist/agent/executor.js +128 -0
  6. package/dist/agent/executor.js.map +1 -0
  7. package/dist/agent/pipeline.d.ts +5 -0
  8. package/dist/agent/pipeline.d.ts.map +1 -0
  9. package/dist/agent/pipeline.js +198 -0
  10. package/dist/agent/pipeline.js.map +1 -0
  11. package/dist/agent/planner.d.ts +9 -0
  12. package/dist/agent/planner.d.ts.map +1 -0
  13. package/dist/agent/planner.js +190 -0
  14. package/dist/agent/planner.js.map +1 -0
  15. package/dist/cache/change-detector.d.ts +7 -0
  16. package/dist/cache/change-detector.d.ts.map +1 -0
  17. package/dist/cache/change-detector.js +43 -0
  18. package/dist/cache/change-detector.js.map +1 -0
  19. package/dist/cache/db.d.ts.map +1 -1
  20. package/dist/cache/db.js +32 -0
  21. package/dist/cache/db.js.map +1 -1
  22. package/dist/cache/diff-summary.d.ts +2 -0
  23. package/dist/cache/diff-summary.d.ts.map +1 -0
  24. package/dist/cache/diff-summary.js +87 -0
  25. package/dist/cache/diff-summary.js.map +1 -0
  26. package/dist/cache/store.d.ts +16 -0
  27. package/dist/cache/store.d.ts.map +1 -1
  28. package/dist/cache/store.js +79 -0
  29. package/dist/cache/store.js.map +1 -1
  30. package/dist/cli/auth.d.ts +2 -0
  31. package/dist/cli/auth.d.ts.map +1 -0
  32. package/dist/cli/auth.js +95 -0
  33. package/dist/cli/auth.js.map +1 -0
  34. package/dist/cli/daemon.d.ts +6 -1
  35. package/dist/cli/daemon.d.ts.map +1 -1
  36. package/dist/cli/daemon.js +56 -3
  37. package/dist/cli/daemon.js.map +1 -1
  38. package/dist/cli/health.d.ts +1 -1
  39. package/dist/cli/health.d.ts.map +1 -1
  40. package/dist/cli/health.js +41 -3
  41. package/dist/cli/health.js.map +1 -1
  42. package/dist/cli/index.d.ts +1 -1
  43. package/dist/cli/index.d.ts.map +1 -1
  44. package/dist/cli/index.js +1 -1
  45. package/dist/cli/index.js.map +1 -1
  46. package/dist/cli/plugin.d.ts +5 -0
  47. package/dist/cli/plugin.d.ts.map +1 -0
  48. package/dist/cli/plugin.js +188 -0
  49. package/dist/cli/plugin.js.map +1 -0
  50. package/dist/cli/shell.d.ts +2 -0
  51. package/dist/cli/shell.d.ts.map +1 -0
  52. package/dist/cli/shell.js +86 -0
  53. package/dist/cli/shell.js.map +1 -0
  54. package/dist/cli/warmup.d.ts +8 -0
  55. package/dist/cli/warmup.d.ts.map +1 -1
  56. package/dist/cli/warmup.js +106 -1
  57. package/dist/cli/warmup.js.map +1 -1
  58. package/dist/config.d.ts +15 -0
  59. package/dist/config.d.ts.map +1 -1
  60. package/dist/config.js +23 -0
  61. package/dist/config.js.map +1 -1
  62. package/dist/daemon/health-check.d.ts +16 -0
  63. package/dist/daemon/health-check.d.ts.map +1 -0
  64. package/dist/daemon/health-check.js +36 -0
  65. package/dist/daemon/health-check.js.map +1 -0
  66. package/dist/daemon/http-server.d.ts +26 -0
  67. package/dist/daemon/http-server.d.ts.map +1 -0
  68. package/dist/daemon/http-server.js +282 -0
  69. package/dist/daemon/http-server.js.map +1 -0
  70. package/dist/daemon/proxy.d.ts +10 -0
  71. package/dist/daemon/proxy.d.ts.map +1 -0
  72. package/dist/daemon/proxy.js +99 -0
  73. package/dist/daemon/proxy.js.map +1 -0
  74. package/dist/embedding/embed.d.ts +19 -0
  75. package/dist/embedding/embed.d.ts.map +1 -0
  76. package/dist/embedding/embed.js +131 -0
  77. package/dist/embedding/embed.js.map +1 -0
  78. package/dist/embedding/key-terms.d.ts +12 -0
  79. package/dist/embedding/key-terms.d.ts.map +1 -0
  80. package/dist/embedding/key-terms.js +138 -0
  81. package/dist/embedding/key-terms.js.map +1 -0
  82. package/dist/embedding/subprocess.d.ts +31 -0
  83. package/dist/embedding/subprocess.d.ts.map +1 -0
  84. package/dist/embedding/subprocess.js +213 -0
  85. package/dist/embedding/subprocess.js.map +1 -0
  86. package/dist/embedding/vector-index.d.ts +26 -0
  87. package/dist/embedding/vector-index.d.ts.map +1 -0
  88. package/dist/embedding/vector-index.js +78 -0
  89. package/dist/embedding/vector-index.js.map +1 -0
  90. package/dist/fetch/action-executor.d.ts +28 -0
  91. package/dist/fetch/action-executor.d.ts.map +1 -0
  92. package/dist/fetch/action-executor.js +86 -0
  93. package/dist/fetch/action-executor.js.map +1 -0
  94. package/dist/fetch/auth.d.ts +2 -1
  95. package/dist/fetch/auth.d.ts.map +1 -1
  96. package/dist/fetch/auth.js +30 -2
  97. package/dist/fetch/auth.js.map +1 -1
  98. package/dist/fetch/browser-pool.d.ts +30 -11
  99. package/dist/fetch/browser-pool.d.ts.map +1 -1
  100. package/dist/fetch/browser-pool.js +228 -51
  101. package/dist/fetch/browser-pool.js.map +1 -1
  102. package/dist/fetch/browser-selector.d.ts +17 -0
  103. package/dist/fetch/browser-selector.d.ts.map +1 -0
  104. package/dist/fetch/browser-selector.js +70 -0
  105. package/dist/fetch/browser-selector.js.map +1 -0
  106. package/dist/fetch/browser-types.d.ts +3 -0
  107. package/dist/fetch/browser-types.d.ts.map +1 -0
  108. package/dist/fetch/browser-types.js +45 -0
  109. package/dist/fetch/browser-types.js.map +1 -0
  110. package/dist/fetch/cdp-client.d.ts +9 -0
  111. package/dist/fetch/cdp-client.d.ts.map +1 -0
  112. package/dist/fetch/cdp-client.js +90 -0
  113. package/dist/fetch/cdp-client.js.map +1 -0
  114. package/dist/fetch/lightpanda.d.ts +28 -0
  115. package/dist/fetch/lightpanda.d.ts.map +1 -0
  116. package/dist/fetch/lightpanda.js +177 -0
  117. package/dist/fetch/lightpanda.js.map +1 -0
  118. package/dist/fetch/router.d.ts +4 -1
  119. package/dist/fetch/router.d.ts.map +1 -1
  120. package/dist/fetch/router.js +8 -2
  121. package/dist/fetch/router.js.map +1 -1
  122. package/dist/index.js +32 -3
  123. package/dist/index.js.map +1 -1
  124. package/dist/instructions.d.ts +29 -0
  125. package/dist/instructions.d.ts.map +1 -0
  126. package/dist/instructions.js +176 -0
  127. package/dist/instructions.js.map +1 -0
  128. package/dist/logger.d.ts +1 -1
  129. package/dist/logger.d.ts.map +1 -1
  130. package/dist/plugins/loader.d.ts +20 -0
  131. package/dist/plugins/loader.d.ts.map +1 -0
  132. package/dist/plugins/loader.js +162 -0
  133. package/dist/plugins/loader.js.map +1 -0
  134. package/dist/plugins/registry.d.ts +26 -0
  135. package/dist/plugins/registry.d.ts.map +1 -0
  136. package/dist/plugins/registry.js +68 -0
  137. package/dist/plugins/registry.js.map +1 -0
  138. package/dist/plugins/validate.d.ts +9 -0
  139. package/dist/plugins/validate.d.ts.map +1 -0
  140. package/dist/plugins/validate.js +70 -0
  141. package/dist/plugins/validate.js.map +1 -0
  142. package/dist/repl/commands/agent.d.ts +5 -0
  143. package/dist/repl/commands/agent.d.ts.map +1 -0
  144. package/dist/repl/commands/agent.js +48 -0
  145. package/dist/repl/commands/agent.js.map +1 -0
  146. package/dist/repl/commands/cache.d.ts +4 -0
  147. package/dist/repl/commands/cache.d.ts.map +1 -0
  148. package/dist/repl/commands/cache.js +44 -0
  149. package/dist/repl/commands/cache.js.map +1 -0
  150. package/dist/repl/commands/crawl.d.ts +7 -0
  151. package/dist/repl/commands/crawl.d.ts.map +1 -0
  152. package/dist/repl/commands/crawl.js +42 -0
  153. package/dist/repl/commands/crawl.js.map +1 -0
  154. package/dist/repl/commands/extract.d.ts +5 -0
  155. package/dist/repl/commands/extract.d.ts.map +1 -0
  156. package/dist/repl/commands/extract.js +37 -0
  157. package/dist/repl/commands/extract.js.map +1 -0
  158. package/dist/repl/commands/fetch.d.ts +5 -0
  159. package/dist/repl/commands/fetch.d.ts.map +1 -0
  160. package/dist/repl/commands/fetch.js +53 -0
  161. package/dist/repl/commands/fetch.js.map +1 -0
  162. package/dist/repl/commands/find-similar.d.ts +5 -0
  163. package/dist/repl/commands/find-similar.d.ts.map +1 -0
  164. package/dist/repl/commands/find-similar.js +61 -0
  165. package/dist/repl/commands/find-similar.js.map +1 -0
  166. package/dist/repl/commands/research.d.ts +5 -0
  167. package/dist/repl/commands/research.d.ts.map +1 -0
  168. package/dist/repl/commands/research.js +50 -0
  169. package/dist/repl/commands/research.js.map +1 -0
  170. package/dist/repl/commands/search.d.ts +5 -0
  171. package/dist/repl/commands/search.d.ts.map +1 -0
  172. package/dist/repl/commands/search.js +62 -0
  173. package/dist/repl/commands/search.js.map +1 -0
  174. package/dist/repl/commands/types.d.ts +9 -0
  175. package/dist/repl/commands/types.d.ts.map +1 -0
  176. package/dist/repl/commands/types.js +2 -0
  177. package/dist/repl/commands/types.js.map +1 -0
  178. package/dist/repl/formatters.d.ts +13 -0
  179. package/dist/repl/formatters.d.ts.map +1 -0
  180. package/dist/repl/formatters.js +282 -0
  181. package/dist/repl/formatters.js.map +1 -0
  182. package/dist/repl/parser.d.ts +9 -0
  183. package/dist/repl/parser.d.ts.map +1 -0
  184. package/dist/repl/parser.js +84 -0
  185. package/dist/repl/parser.js.map +1 -0
  186. package/dist/repl/shell.d.ts +8 -0
  187. package/dist/repl/shell.d.ts.map +1 -0
  188. package/dist/repl/shell.js +177 -0
  189. package/dist/repl/shell.js.map +1 -0
  190. package/dist/research/decompose.d.ts +7 -0
  191. package/dist/research/decompose.d.ts.map +1 -0
  192. package/dist/research/decompose.js +195 -0
  193. package/dist/research/decompose.js.map +1 -0
  194. package/dist/research/pipeline.d.ts +5 -0
  195. package/dist/research/pipeline.d.ts.map +1 -0
  196. package/dist/research/pipeline.js +135 -0
  197. package/dist/research/pipeline.js.map +1 -0
  198. package/dist/research/synthesize.d.ts +10 -0
  199. package/dist/research/synthesize.d.ts.map +1 -0
  200. package/dist/research/synthesize.js +119 -0
  201. package/dist/research/synthesize.js.map +1 -0
  202. package/dist/search/answer-synthesis.d.ts +13 -0
  203. package/dist/search/answer-synthesis.d.ts.map +1 -0
  204. package/dist/search/answer-synthesis.js +120 -0
  205. package/dist/search/answer-synthesis.js.map +1 -0
  206. package/dist/search/context-formatter.d.ts +3 -0
  207. package/dist/search/context-formatter.d.ts.map +1 -0
  208. package/dist/search/context-formatter.js +56 -0
  209. package/dist/search/context-formatter.js.map +1 -0
  210. package/dist/search/find-similar.d.ts +5 -0
  211. package/dist/search/find-similar.d.ts.map +1 -0
  212. package/dist/search/find-similar.js +329 -0
  213. package/dist/search/find-similar.js.map +1 -0
  214. package/dist/search/multi-query.d.ts +22 -0
  215. package/dist/search/multi-query.d.ts.map +1 -0
  216. package/dist/search/multi-query.js +157 -0
  217. package/dist/search/multi-query.js.map +1 -0
  218. package/dist/search/rrf.d.ts +17 -0
  219. package/dist/search/rrf.d.ts.map +1 -0
  220. package/dist/search/rrf.js +48 -0
  221. package/dist/search/rrf.js.map +1 -0
  222. package/dist/search/sampling.d.ts +25 -0
  223. package/dist/search/sampling.d.ts.map +1 -0
  224. package/dist/search/sampling.js +52 -0
  225. package/dist/search/sampling.js.map +1 -0
  226. package/dist/server.d.ts +17 -0
  227. package/dist/server.d.ts.map +1 -1
  228. package/dist/server.js +366 -105
  229. package/dist/server.js.map +1 -1
  230. package/dist/tools/agent.d.ts +5 -0
  231. package/dist/tools/agent.d.ts.map +1 -0
  232. package/dist/tools/agent.js +67 -0
  233. package/dist/tools/agent.js.map +1 -0
  234. package/dist/tools/cache.d.ts +2 -1
  235. package/dist/tools/cache.d.ts.map +1 -1
  236. package/dist/tools/cache.js +56 -1
  237. package/dist/tools/cache.js.map +1 -1
  238. package/dist/tools/fetch.d.ts.map +1 -1
  239. package/dist/tools/fetch.js +26 -1
  240. package/dist/tools/fetch.js.map +1 -1
  241. package/dist/tools/find-similar.d.ts +5 -0
  242. package/dist/tools/find-similar.d.ts.map +1 -0
  243. package/dist/tools/find-similar.js +48 -0
  244. package/dist/tools/find-similar.js.map +1 -0
  245. package/dist/tools/research.d.ts +5 -0
  246. package/dist/tools/research.d.ts.map +1 -0
  247. package/dist/tools/research.js +50 -0
  248. package/dist/tools/research.js.map +1 -0
  249. package/dist/tools/search.d.ts +2 -1
  250. package/dist/tools/search.d.ts.map +1 -1
  251. package/dist/tools/search.js +179 -13
  252. package/dist/tools/search.js.map +1 -1
  253. package/dist/types.d.ts +147 -2
  254. package/dist/types.d.ts.map +1 -1
  255. package/package.json +43 -4
@@ -0,0 +1,52 @@
1
+ import { createLogger } from '../logger.js';
2
+ const log = createLogger('search');
3
+ export function checkSamplingSupport(server) {
4
+ try {
5
+ const capabilities = server.getClientCapabilities();
6
+ if (!capabilities)
7
+ return false;
8
+ return capabilities.sampling !== undefined;
9
+ }
10
+ catch (err) {
11
+ log.debug('sampling support check failed', { error: String(err) });
12
+ return false;
13
+ }
14
+ }
15
+ export async function requestSampling(server, messages, maxTokens) {
16
+ if (!checkSamplingSupport(server)) {
17
+ throw new Error('Client does not support MCP sampling');
18
+ }
19
+ log.debug('sending sampling request', {
20
+ messageCount: messages.length,
21
+ maxTokens,
22
+ });
23
+ const response = await server.createMessage({
24
+ messages,
25
+ maxTokens,
26
+ });
27
+ log.debug('sampling response received', {
28
+ model: response?.model,
29
+ contentType: response?.content?.type,
30
+ textLength: response?.content?.text?.length ?? 0,
31
+ });
32
+ return response;
33
+ }
34
+ export function extractTextFromSamplingResponse(response) {
35
+ try {
36
+ if (!response || typeof response !== 'object')
37
+ return '';
38
+ const resp = response;
39
+ const content = resp.content;
40
+ if (!content)
41
+ return '';
42
+ if (content.type === 'text' && typeof content.text === 'string') {
43
+ return content.text.trim();
44
+ }
45
+ return '';
46
+ }
47
+ catch (err) {
48
+ log.debug('failed to extract text from sampling response', { error: String(err) });
49
+ return '';
50
+ }
51
+ }
52
+ //# sourceMappingURL=sampling.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sampling.js","sourceRoot":"","sources":["../../src/search/sampling.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAoBnC,MAAM,UAAU,oBAAoB,CAAC,MAA6B;IAChE,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY;YAAE,OAAO,KAAK,CAAC;QAChC,OAAO,YAAY,CAAC,QAAQ,KAAK,SAAS,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnE,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAA6B,EAC7B,QAA2B,EAC3B,SAAiB;IAEjB,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE;QACpC,YAAY,EAAE,QAAQ,CAAC,MAAM;QAC7B,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;QAC1C,QAAQ;QACR,SAAS;KACV,CAAC,CAAC;IAEH,GAAG,CAAC,KAAK,CAAC,4BAA4B,EAAE;QACtC,KAAK,EAAE,QAAQ,EAAE,KAAK;QACtB,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI;QACpC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;KACjD,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,QAAiB;IAEjB,IAAI,CAAC;QACH,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAEzD,MAAM,IAAI,GAAG,QAAmC,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAqD,CAAC;QAC3E,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAChE,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7B,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
package/dist/server.d.ts CHANGED
@@ -1,2 +1,19 @@
1
+ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
2
+ import { SmartRouter } from './fetch/router.js';
3
+ import { MultiBrowserPool } from './fetch/browser-pool.js';
4
+ import { BackendStatus } from './server/backend-status.js';
5
+ import { PluginRegistry } from './plugins/registry.js';
6
+ import type { SearchEngine } from './types.js';
7
+ export interface Subsystems {
8
+ searchEngines: SearchEngine[];
9
+ browserPool: MultiBrowserPool;
10
+ router: SmartRouter;
11
+ backendStatus: BackendStatus;
12
+ pluginRegistry: PluginRegistry;
13
+ shutdown: () => Promise<void>;
14
+ bootstrapSearxng: () => Promise<void>;
15
+ }
16
+ export declare function initSubsystems(): Promise<Subsystems>;
17
+ export declare function createMcpServer(subsystems: Subsystems): Server;
1
18
  export declare function startServer(): Promise<void>;
2
19
  //# sourceMappingURL=server.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAsMA,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CA8OjD"}
1
+ {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAMnE,OAAO,EAAE,WAAW,EAAmB,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAmB3D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAK3D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,KAAK,EAA2B,YAAY,EAAqF,MAAM,YAAY,CAAC;AAiV3J,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,WAAW,EAAE,gBAAgB,CAAC;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,aAAa,EAAE,aAAa,CAAC;IAC7B,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,gBAAgB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAED,wBAAsB,cAAc,IAAI,OAAO,CAAC,UAAU,CAAC,CAqK1D;AAED,wBAAgB,eAAe,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAkJ9D;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAoBjD"}
package/dist/server.js CHANGED
@@ -5,7 +5,7 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
5
5
  import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
6
6
  import { ListToolsRequestSchema, CallToolRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
7
7
  import { SmartRouter } from './fetch/router.js';
8
- import { BrowserPool } from './fetch/browser-pool.js';
8
+ import { MultiBrowserPool } from './fetch/browser-pool.js';
9
9
  import { httpFetch } from './fetch/http-client.js';
10
10
  import { initDatabase, closeDatabase } from './cache/db.js';
11
11
  import { handleFetch } from './tools/fetch.js';
@@ -13,6 +13,9 @@ import { handleSearch } from './tools/search.js';
13
13
  import { handleCrawl } from './tools/crawl.js';
14
14
  import { handleCache } from './tools/cache.js';
15
15
  import { handleExtract } from './tools/extract.js';
16
+ import { handleFindSimilar } from './tools/find-similar.js';
17
+ import { handleResearch } from './tools/research.js';
18
+ import { handleAgent } from './tools/agent.js';
16
19
  import { SearxngClient } from './search/searxng.js';
17
20
  import { DuckDuckGoEngine } from './search/engines/duckduckgo.js';
18
21
  import { BingEngine } from './search/engines/bing.js';
@@ -23,6 +26,10 @@ import { DockerSearxng } from './searxng/docker.js';
23
26
  import { BackendStatus } from './server/backend-status.js';
24
27
  import { getConfig } from './config.js';
25
28
  import { createLogger } from './logger.js';
29
+ import { WIGOLO_INSTRUCTIONS, TOOL_DESCRIPTIONS } from './instructions.js';
30
+ import { loadPlugins } from './plugins/loader.js';
31
+ import { PluginRegistry } from './plugins/registry.js';
32
+ import { registerExtractor } from './extraction/pipeline.js';
26
33
  const log = createLogger('server');
27
34
  function readPackageVersion() {
28
35
  try {
@@ -71,13 +78,64 @@ const FETCH_TOOL_SCHEMA = {
71
78
  description: 'Additional HTTP headers',
72
79
  additionalProperties: { type: 'string' },
73
80
  },
81
+ actions: {
82
+ type: 'array',
83
+ description: 'Sequential browser actions to perform before extracting content. ' +
84
+ 'When present, forces Playwright rendering (bypasses HTTP-first routing).',
85
+ items: {
86
+ type: 'object',
87
+ properties: {
88
+ type: {
89
+ type: 'string',
90
+ enum: ['click', 'type', 'wait', 'wait_for', 'scroll', 'screenshot'],
91
+ description: 'Action type',
92
+ },
93
+ selector: {
94
+ type: 'string',
95
+ description: 'CSS selector (required for click, type, wait_for)',
96
+ },
97
+ text: {
98
+ type: 'string',
99
+ description: 'Text to type (required for type action)',
100
+ },
101
+ ms: {
102
+ type: 'number',
103
+ description: 'Milliseconds to wait (required for wait action)',
104
+ },
105
+ timeout: {
106
+ type: 'number',
107
+ description: 'Timeout in ms for wait_for action (default: 5000)',
108
+ },
109
+ direction: {
110
+ type: 'string',
111
+ enum: ['down', 'up'],
112
+ description: 'Scroll direction (required for scroll action)',
113
+ },
114
+ amount: {
115
+ type: 'number',
116
+ description: 'Scroll amount in pixels (default: viewport height)',
117
+ },
118
+ },
119
+ required: ['type'],
120
+ },
121
+ },
74
122
  },
75
123
  required: ['url'],
76
124
  };
77
125
  const SEARCH_TOOL_SCHEMA = {
78
126
  type: 'object',
79
127
  properties: {
80
- query: { type: 'string', description: 'Search query' },
128
+ query: {
129
+ oneOf: [
130
+ { type: 'string', description: 'Search query' },
131
+ {
132
+ type: 'array',
133
+ items: { type: 'string' },
134
+ description: 'Array of query variants to search in parallel, deduplicate, and rerank',
135
+ },
136
+ ],
137
+ description: 'Search query — a single string or array of query variants for parallel multi-query search',
138
+ },
81
139
  max_results: { type: 'number', description: 'Max results to return (default 5, max 20)' },
82
140
  include_content: { type: 'boolean', description: 'Fetch full content for results (default true)' },
83
141
  content_max_chars: { type: 'number', description: 'Max chars per result content (default 30000)' },
@@ -108,6 +166,11 @@ const SEARCH_TOOL_SCHEMA = {
108
166
  enum: ['general', 'news', 'code', 'docs', 'papers', 'images'],
109
167
  description: 'Category of search (general, news, code, docs, papers, images)',
110
168
  },
169
+ format: {
170
+ type: 'string',
171
+ enum: ['full', 'context', 'answer', 'stream_answer'],
172
+ description: "Output format: 'full' returns structured results (default), 'context' returns a single token-budgeted string for LLM injection, 'answer' synthesizes a direct answer with citations via MCP sampling, 'stream_answer' same as answer with streaming flag",
173
+ },
111
174
  },
112
175
  required: ['query'],
113
176
  };
@@ -158,6 +221,12 @@ const CACHE_TOOL_SCHEMA = {
158
221
  type: 'boolean',
159
222
  description: 'Return cache statistics (total URLs, size, date range)',
160
223
  },
224
+ check_changes: {
225
+ type: 'boolean',
226
+ description: 'Re-fetch all matching cached URLs and report which ones have changed. ' +
227
+ 'Returns a list of URLs with changed/unchanged status and diff summaries. ' +
228
+ 'Use with query or url_pattern to scope which cached entries to check.',
229
+ },
161
230
  },
162
231
  };
163
232
  const EXTRACT_TOOL_SCHEMA = {
@@ -184,123 +253,161 @@ const EXTRACT_TOOL_SCHEMA = {
184
253
  },
185
254
  },
186
255
  };
187
- export async function startServer() {
256
+ const FIND_SIMILAR_TOOL_SCHEMA = {
257
+ type: 'object',
258
+ properties: {
259
+ url: {
260
+ type: 'string',
261
+ description: 'Find pages similar to this URL. The page is fetched (or read from cache) and its content analyzed for key terms.',
262
+ },
263
+ concept: {
264
+ type: 'string',
265
+ description: 'Find pages related to this concept or topic description. Use when you don\'t have a specific URL.',
266
+ },
267
+ max_results: {
268
+ type: 'number',
269
+ description: 'Maximum results to return (default 10, max 50)',
270
+ },
271
+ include_domains: {
272
+ type: 'array',
273
+ items: { type: 'string' },
274
+ description: 'Only return results from these domains',
275
+ },
276
+ exclude_domains: {
277
+ type: 'array',
278
+ items: { type: 'string' },
279
+ description: 'Never return results from these domains',
280
+ },
281
+ include_cache: {
282
+ type: 'boolean',
283
+ description: 'Search local cache for similar pages (default: true)',
284
+ },
285
+ include_web: {
286
+ type: 'boolean',
287
+ description: 'Supplement with web search if needed (default: true)',
288
+ },
289
+ },
290
+ };
291
+ const RESEARCH_TOOL_SCHEMA = {
292
+ type: 'object',
293
+ properties: {
294
+ question: { type: 'string', description: 'The research question to investigate' },
295
+ depth: {
296
+ type: 'string',
297
+ enum: ['quick', 'standard', 'comprehensive'],
298
+ description: 'Research depth: quick (~15s), standard (~40s, default), comprehensive (~80s)',
299
+ },
300
+ max_sources: {
301
+ type: 'number',
302
+ description: 'Override the default source count for the chosen depth (max 50)',
303
+ },
304
+ include_domains: {
305
+ type: 'array',
306
+ items: { type: 'string' },
307
+ description: 'Only search results from these domains',
308
+ },
309
+ exclude_domains: {
310
+ type: 'array',
311
+ items: { type: 'string' },
312
+ description: 'Exclude results from these domains',
313
+ },
314
+ schema: {
315
+ type: 'object',
316
+ description: 'Optional JSON Schema -- structure the report to extract these fields',
317
+ },
318
+ stream: {
319
+ type: 'boolean',
320
+ description: 'Send progress notifications as each research phase completes',
321
+ },
322
+ },
323
+ required: ['question'],
324
+ };
325
+ const AGENT_TOOL_SCHEMA = {
326
+ type: 'object',
327
+ properties: {
328
+ prompt: {
329
+ type: 'string',
330
+ description: 'Natural-language description of what data to gather',
331
+ },
332
+ urls: {
333
+ type: 'array',
334
+ items: { type: 'string' },
335
+ description: 'Specific URLs to include in the data gathering',
336
+ },
337
+ schema: {
338
+ type: 'object',
339
+ description: 'Optional JSON Schema -- extract structured data matching this schema from each page',
340
+ },
341
+ max_pages: {
342
+ type: 'number',
343
+ description: 'Maximum pages to fetch (default 10, max 100)',
344
+ },
345
+ max_time_ms: {
346
+ type: 'number',
347
+ description: 'Maximum execution time in milliseconds (default 60000)',
348
+ },
349
+ stream: {
350
+ type: 'boolean',
351
+ description: 'Send progress notifications as each step completes',
352
+ },
353
+ },
354
+ required: ['prompt'],
355
+ };
356
+ export async function initSubsystems() {
188
357
  const config = getConfig();
189
358
  mkdirSync(config.dataDir, { recursive: true });
190
359
  initDatabase(join(config.dataDir, 'wigolo.db'));
191
360
  const httpClient = {
192
361
  fetch: (url, options) => httpFetch(url, options),
193
362
  };
194
- const browserPool = new BrowserPool();
363
+ const browserPool = new MultiBrowserPool({
364
+ browserTypes: config.browserTypes,
365
+ selectionStrategy: 'round-robin',
366
+ });
195
367
  const router = new SmartRouter(httpClient, browserPool);
368
+ if (config.lightpandaEnabled && config.lightpandaUrl) {
369
+ log.info('lightpanda browser backend enabled', {
370
+ url: config.lightpandaUrl,
371
+ failureThreshold: config.lightpandaFailureThreshold,
372
+ });
373
+ }
196
374
  const backendStatus = new BackendStatus();
197
- // Direct scraping engines work without any bootstrap — always available so
198
- // search() succeeds immediately even before SearXNG finishes setup.
199
375
  const searchEngines = [
200
376
  new BingEngine(),
201
377
  new DuckDuckGoEngine(),
202
378
  new StartpageEngine(),
203
379
  ];
204
- let searxngProcess = null;
205
- let dockerSearxng = null;
206
- const server = new Server({ name: 'wigolo', version: SERVER_VERSION }, { capabilities: { tools: {} } });
207
- server.setRequestHandler(ListToolsRequestSchema, async () => ({
208
- tools: [
209
- {
210
- name: 'fetch',
211
- description: 'Fetch a web page and return its content as clean markdown. ' +
212
- 'Supports JavaScript rendering, auth, section extraction, and caching.',
213
- inputSchema: FETCH_TOOL_SCHEMA,
214
- },
215
- {
216
- name: 'search',
217
- description: 'Search the web and return results with optional full content extraction. ' +
218
- 'One call: query in, clean markdown out.',
219
- inputSchema: SEARCH_TOOL_SCHEMA,
220
- },
221
- {
222
- name: 'crawl',
223
- description: 'Crawl a website starting from a seed URL. Supports BFS, DFS, and sitemap strategies ' +
224
- 'with depth/page limits, URL filtering, and cross-page content deduplication.',
225
- inputSchema: CRAWL_TOOL_SCHEMA,
226
- },
227
- {
228
- name: 'cache',
229
- description: 'Query the local knowledge base of previously fetched content. ' +
230
- 'Search cached pages by full-text query, URL pattern, or date. ' +
231
- 'Can also return cache statistics or clear entries.',
232
- inputSchema: CACHE_TOOL_SCHEMA,
233
- },
234
- {
235
- name: 'extract',
236
- description: 'Extract structured data from a web page. Supports CSS selector extraction, ' +
237
- 'table-to-JSON conversion, metadata extraction (title, author, date, JSON-LD), ' +
238
- 'and schema-based extraction (provide a JSON Schema to heuristically extract matching fields).',
239
- inputSchema: EXTRACT_TOOL_SCHEMA,
240
- },
241
- ],
242
- }));
243
- server.setRequestHandler(CallToolRequestSchema, async (request) => {
244
- const { name, arguments: args } = request.params;
245
- if (name === 'fetch') {
246
- const input = (args ?? {});
247
- const result = await handleFetch(input, router);
248
- return {
249
- content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
250
- isError: !!result.error,
251
- };
380
+ // Load plugins from ~/.wigolo/plugins/
381
+ const pluginRegistry = new PluginRegistry();
382
+ try {
383
+ const pluginResult = await loadPlugins();
384
+ for (const ext of pluginResult.extractors) {
385
+ pluginRegistry.registerExtractor(ext, ext.name);
386
+ registerExtractor(ext);
252
387
  }
253
- if (name === 'search') {
254
- const input = (args ?? {});
255
- const result = await handleSearch(input, searchEngines, router, backendStatus);
256
- const blocks = [];
257
- if (result.warning) {
258
- blocks.push({ type: 'text', text: `[wigolo notice] ${result.warning}` });
259
- }
260
- blocks.push({ type: 'text', text: JSON.stringify(result, null, 2) });
261
- return {
262
- content: blocks,
263
- isError: !!result.error,
264
- };
388
+ for (const eng of pluginResult.searchEngines) {
389
+ pluginRegistry.registerSearchEngine(eng, eng.name);
390
+ searchEngines.push(eng);
265
391
  }
266
- if (name === 'crawl') {
267
- const input = (args ?? {});
268
- const result = await handleCrawl(input, router);
269
- return {
270
- content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
271
- isError: !!result.error,
272
- };
392
+ if (pluginResult.errors.length > 0) {
393
+ log.warn('some plugins failed to load', {
394
+ errors: pluginResult.errors.map(e => `${e.pluginName}: ${e.message}`),
395
+ });
273
396
  }
274
- if (name === 'cache') {
275
- const input = (args ?? {});
276
- const result = handleCache(input);
277
- return {
278
- content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
279
- isError: !!result.error,
280
- };
397
+ if (pluginResult.loaded.length > 0) {
398
+ log.info('plugins loaded', {
399
+ count: pluginResult.loaded.length,
400
+ names: pluginResult.loaded.map(p => p.name),
401
+ });
281
402
  }
282
- if (name === 'extract') {
283
- const input = (args ?? {});
284
- const result = await handleExtract(input, router);
285
- return {
286
- content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
287
- isError: !!result.error,
288
- };
289
- }
290
- return {
291
- content: [{ type: 'text', text: `Unknown tool: ${name}` }],
292
- isError: true,
293
- };
294
- });
295
- const transport = new StdioServerTransport();
296
- await server.connect(transport);
297
- log.info('MCP server started');
298
- // Bootstrap SearXNG in the background. The MCP server is already responding
299
- // to initialize/tools/list/tools/call. search() falls back to direct engines
300
- // until SearXNG becomes available, at which point it is unshifted to the
301
- // front of the engine list and used preferentially on subsequent calls.
403
+ }
404
+ catch (err) {
405
+ log.error('plugin loading failed', { error: String(err) });
406
+ }
407
+ let searxngProcess = null;
408
+ let dockerSearxng = null;
302
409
  let searxngBootstrap = null;
303
- searxngBootstrap = (async () => {
410
+ async function bootstrapSearxng() {
304
411
  try {
305
412
  const backend = await resolveSearchBackend();
306
413
  if (backend.type === 'external' && backend.url) {
@@ -379,12 +486,10 @@ export async function startServer() {
379
486
  log.warn('background backend setup failed', { error: String(err) });
380
487
  backendStatus.markUnhealthy(`backend setup failed: ${String(err)}`);
381
488
  }
382
- })();
383
- const shutdown = async () => {
489
+ }
490
+ async function shutdown() {
384
491
  log.info('Shutting down');
385
492
  if (searxngBootstrap) {
386
- // Let an in-flight bootstrap settle so we don't kill venv/pip mid-install
387
- // and corrupt ~/.wigolo/searxng. Bounded so we don't hang forever.
388
493
  await Promise.race([
389
494
  searxngBootstrap.catch(() => { }),
390
495
  new Promise((r) => setTimeout(r, 2000)),
@@ -396,6 +501,162 @@ export async function startServer() {
396
501
  await dockerSearxng.stop();
397
502
  await browserPool.shutdown();
398
503
  closeDatabase();
504
+ }
505
+ return {
506
+ searchEngines,
507
+ browserPool,
508
+ router,
509
+ backendStatus,
510
+ pluginRegistry,
511
+ shutdown,
512
+ bootstrapSearxng: () => {
513
+ searxngBootstrap = bootstrapSearxng();
514
+ return searxngBootstrap;
515
+ },
516
+ };
517
+ }
518
+ export function createMcpServer(subsystems) {
519
+ const { searchEngines, router, backendStatus } = subsystems;
520
+ const server = new Server({ name: 'wigolo', version: SERVER_VERSION }, {
521
+ capabilities: { tools: {} },
522
+ instructions: WIGOLO_INSTRUCTIONS,
523
+ });
524
+ server.setRequestHandler(ListToolsRequestSchema, async () => ({
525
+ tools: [
526
+ {
527
+ name: 'fetch',
528
+ description: TOOL_DESCRIPTIONS.fetch,
529
+ inputSchema: FETCH_TOOL_SCHEMA,
530
+ },
531
+ {
532
+ name: 'search',
533
+ description: TOOL_DESCRIPTIONS.search,
534
+ inputSchema: SEARCH_TOOL_SCHEMA,
535
+ },
536
+ {
537
+ name: 'crawl',
538
+ description: TOOL_DESCRIPTIONS.crawl,
539
+ inputSchema: CRAWL_TOOL_SCHEMA,
540
+ },
541
+ {
542
+ name: 'cache',
543
+ description: TOOL_DESCRIPTIONS.cache,
544
+ inputSchema: CACHE_TOOL_SCHEMA,
545
+ },
546
+ {
547
+ name: 'extract',
548
+ description: TOOL_DESCRIPTIONS.extract,
549
+ inputSchema: EXTRACT_TOOL_SCHEMA,
550
+ },
551
+ {
552
+ name: 'find_similar',
553
+ description: TOOL_DESCRIPTIONS.find_similar,
554
+ inputSchema: FIND_SIMILAR_TOOL_SCHEMA,
555
+ },
556
+ {
557
+ name: 'research',
558
+ description: TOOL_DESCRIPTIONS.research,
559
+ inputSchema: RESEARCH_TOOL_SCHEMA,
560
+ },
561
+ {
562
+ name: 'agent',
563
+ description: TOOL_DESCRIPTIONS.agent,
564
+ inputSchema: AGENT_TOOL_SCHEMA,
565
+ },
566
+ ],
567
+ }));
568
+ server.setRequestHandler(CallToolRequestSchema, async (request) => {
569
+ const { name, arguments: args } = request.params;
570
+ if (name === 'fetch') {
571
+ const input = (args ?? {});
572
+ const result = await handleFetch(input, router);
573
+ return {
574
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
575
+ isError: !!result.error,
576
+ };
577
+ }
578
+ if (name === 'search') {
579
+ const input = (args ?? {});
580
+ const samplingServer = server;
581
+ const result = await handleSearch(input, searchEngines, router, backendStatus, samplingServer);
582
+ const blocks = [];
583
+ if (result.warning) {
584
+ blocks.push({ type: 'text', text: `[wigolo notice] ${result.warning}` });
585
+ }
586
+ blocks.push({ type: 'text', text: JSON.stringify(result, null, 2) });
587
+ return {
588
+ content: blocks,
589
+ isError: !!result.error,
590
+ };
591
+ }
592
+ if (name === 'crawl') {
593
+ const input = (args ?? {});
594
+ const result = await handleCrawl(input, router);
595
+ return {
596
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
597
+ isError: !!result.error,
598
+ };
599
+ }
600
+ if (name === 'cache') {
601
+ const input = (args ?? {});
602
+ const result = await handleCache(input, router);
603
+ return {
604
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
605
+ isError: !!result.error,
606
+ };
607
+ }
608
+ if (name === 'extract') {
609
+ const input = (args ?? {});
610
+ const result = await handleExtract(input, router);
611
+ return {
612
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
613
+ isError: !!result.error,
614
+ };
615
+ }
616
+ if (name === 'find_similar') {
617
+ const input = (args ?? {});
618
+ const result = await handleFindSimilar(input, searchEngines, router, backendStatus);
619
+ return {
620
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
621
+ isError: !!result.error,
622
+ };
623
+ }
624
+ if (name === 'research') {
625
+ const input = (args ?? {});
626
+ const samplingServer = server;
627
+ const result = await handleResearch(input, searchEngines, router, backendStatus, samplingServer);
628
+ return {
629
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
630
+ isError: !!result.error,
631
+ };
632
+ }
633
+ if (name === 'agent') {
634
+ const input = (args ?? {});
635
+ const samplingServer = server;
636
+ const result = await handleAgent(input, searchEngines, router, backendStatus, samplingServer);
637
+ return {
638
+ content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
639
+ isError: !!result.error,
640
+ };
641
+ }
642
+ return {
643
+ content: [{ type: 'text', text: `Unknown tool: ${name}` }],
644
+ isError: true,
645
+ };
646
+ });
647
+ return server;
648
+ }
649
+ export async function startServer() {
650
+ const subs = await initSubsystems();
651
+ const server = createMcpServer(subs);
652
+ const transport = new StdioServerTransport();
653
+ await server.connect(transport);
654
+ log.info('MCP server started');
655
+ subs.bootstrapSearxng().catch((err) => {
656
+ log.warn('SearXNG bootstrap failed', { error: String(err) });
657
+ });
658
+ const shutdown = async () => {
659
+ await subs.shutdown();
399
660
  await server.close();
400
661
  process.exit(0);
401
662
  };