html2rss 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +82 -25
  3. data/html2rss.gemspec +0 -1
  4. data/lib/html2rss/article.rb +1 -2
  5. data/lib/html2rss/auto_source/README.md +8 -8
  6. data/lib/html2rss/auto_source/cleanup.rb +92 -1
  7. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
  8. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
  9. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
  10. data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
  11. data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
  12. data/lib/html2rss/auto_source/scraper.rb +2 -3
  13. data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
  14. data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
  15. data/lib/html2rss/auto_source/segmenter.rb +1 -2
  16. data/lib/html2rss/auto_source.rb +25 -11
  17. data/lib/html2rss/batch.rb +169 -0
  18. data/lib/html2rss/capture/README.md +15 -6
  19. data/lib/html2rss/capture.rb +158 -15
  20. data/lib/html2rss/channel.rb +1 -2
  21. data/lib/html2rss/cli/probe_view.rb +66 -0
  22. data/lib/html2rss/cli/render.rb +124 -0
  23. data/lib/html2rss/cli/validate.rb +91 -0
  24. data/lib/html2rss/cli.rb +318 -193
  25. data/lib/html2rss/config/request_headers.rb +4 -14
  26. data/lib/html2rss/config/schema.rb +7 -5
  27. data/lib/html2rss/config/validator.rb +1 -0
  28. data/lib/html2rss/config.rb +59 -22
  29. data/lib/html2rss/doctor/botasaurus.rb +130 -0
  30. data/lib/html2rss/feed_builder/rss.rb +1 -2
  31. data/lib/html2rss/feed_pipeline/README.md +5 -5
  32. data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
  33. data/lib/html2rss/feed_pipeline.rb +27 -16
  34. data/lib/html2rss/feed_resolution/README.md +11 -11
  35. data/lib/html2rss/feed_resolution.rb +6 -12
  36. data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
  37. data/lib/html2rss/html/article_extractor.rb +4 -9
  38. data/lib/html2rss/html/feed_link.rb +4 -2
  39. data/lib/html2rss/html/navigator.rb +4 -39
  40. data/lib/html2rss/html/probe.rb +82 -0
  41. data/lib/html2rss/html/sst_article_extractor.rb +1 -2
  42. data/lib/html2rss/link_destination/noise_policy.rb +1 -2
  43. data/lib/html2rss/link_destination/path_classifier.rb +20 -19
  44. data/lib/html2rss/mcp/README.md +80 -0
  45. data/lib/html2rss/mcp/contract.rb +135 -13
  46. data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
  47. data/lib/html2rss/mcp/outcome.rb +112 -42
  48. data/lib/html2rss/mcp/runtime.rb +45 -0
  49. data/lib/html2rss/mcp/server/tools.rb +277 -0
  50. data/lib/html2rss/mcp/server.rb +91 -232
  51. data/lib/html2rss/mcp.rb +3 -0
  52. data/lib/html2rss/page_recon/README.md +69 -0
  53. data/lib/html2rss/page_recon/diagnostics.rb +211 -0
  54. data/lib/html2rss/page_recon.rb +56 -2
  55. data/lib/html2rss/recon.rb +252 -0
  56. data/lib/html2rss/request_service/blocked_surface.rb +1 -0
  57. data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
  58. data/lib/html2rss/request_service/policy.rb +1 -2
  59. data/lib/html2rss/request_service/response.rb +3 -4
  60. data/lib/html2rss/request_service/strategy.rb +1 -2
  61. data/lib/html2rss/request_service.rb +2 -0
  62. data/lib/html2rss/request_session.rb +1 -2
  63. data/lib/html2rss/scrape_target.rb +1 -1
  64. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
  65. data/lib/html2rss/selectors/post_processors.rb +0 -1
  66. data/lib/html2rss/selectors.rb +33 -10
  67. data/lib/html2rss/sst/attrs.rb +1 -2
  68. data/lib/html2rss/sst/normalizer.rb +49 -11
  69. data/lib/html2rss/status.rb +3 -6
  70. data/lib/html2rss/syndication/README.md +6 -6
  71. data/lib/html2rss/syndication/discovery.rb +2 -4
  72. data/lib/html2rss/test/enhance_audit.rb +192 -0
  73. data/lib/html2rss/test.rb +458 -0
  74. data/lib/html2rss/url.rb +1 -2
  75. data/lib/html2rss/version.rb +1 -1
  76. data/lib/html2rss.rb +157 -35
  77. data/schema/html2rss-config.schema.json +9 -29
  78. metadata +17 -18
  79. data/lib/html2rss/mcp/inspect.rb +0 -138
  80. data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
@@ -0,0 +1,277 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ module MCP
5
+ module Server
6
+ ##
7
+ # MCP tool registration and outcome mapping over public html2rss APIs.
8
+ module Tools # rubocop:disable Metrics/ModuleLength -- declarative registry + substantive handlers
9
+ # Declarative MCP tool registrations consumed by {register_all}.
10
+ TOOLS = [
11
+ {
12
+ name: 'scrape',
13
+ kind: :url,
14
+ description: 'One-shot article extraction as JSON Feed items. ' \
15
+ 'Use when you need articles now without a saved config. ' \
16
+ 'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
17
+ input_schema: Contract::SCRAPE_INPUT_SCHEMA,
18
+ handler: :scrape_outcome
19
+ },
20
+ {
21
+ name: 'inspect',
22
+ kind: :url,
23
+ description: 'Diagnostic page analysis (scrapers, SST, segments, final URL, status, ' \
24
+ 'rel=alternate feeds). Use recon for BUILD/DEFER/DROP verdict and native_feed preference.',
25
+ input_schema: Contract::INSPECT_INPUT_SCHEMA,
26
+ call: lambda { |url:, strategy: 'auto', **|
27
+ Outcome.inspect(
28
+ report: PageRecon::Diagnostics.call(
29
+ url:, strategy: Runtime.coerce_strategy(strategy), deep: false
30
+ )
31
+ )
32
+ }
33
+ },
34
+ {
35
+ name: 'recon',
36
+ kind: :url,
37
+ description: 'Curation verdict and native_feed preference for a URL. ' \
38
+ 'Use after inspect when alternates warrant deeper recon, or when you need BUILD/DEFER/DROP.',
39
+ input_schema: Contract::RECON_INPUT_SCHEMA,
40
+ call: lambda { |url:, strategy: 'auto', **|
41
+ Outcome.recon(result: Html2rss.recon(url, strategy: Runtime.coerce_strategy(strategy)))
42
+ }
43
+ },
44
+ {
45
+ name: 'batch_scrape',
46
+ kind: :batch,
47
+ batch_method: :batch_scrape,
48
+ limit_default: 10,
49
+ description: 'Scrape multiple URLs in parallel with per-URL error isolation. ' \
50
+ 'Returns structured JSON Feed items and extraction counts.',
51
+ input_schema: Contract::BATCH_SCRAPE_INPUT_SCHEMA
52
+ },
53
+ {
54
+ name: 'batch_inspect',
55
+ kind: :batch,
56
+ batch_method: :batch_inspect,
57
+ description: 'Inspect multiple URLs in parallel with per-URL error isolation. ' \
58
+ 'Returns final redirected URLs, status codes, and rel="alternate" feeds.',
59
+ input_schema: Contract::BATCH_INSPECT_INPUT_SCHEMA
60
+ },
61
+ {
62
+ name: 'batch_recon',
63
+ kind: :batch,
64
+ batch_method: :batch_recon,
65
+ description: 'Run recon across multiple URLs in parallel with per-URL error isolation. ' \
66
+ 'Returns verdict, native_feed, and surface classification per URL.',
67
+ input_schema: Contract::BATCH_RECON_INPUT_SCHEMA
68
+ },
69
+ {
70
+ name: 'capture',
71
+ kind: :capture,
72
+ description: 'Derive a reusable html2rss feed config from a URL. ' \
73
+ 'Use when the goal is a durable YAML (then test → apply). ' \
74
+ 'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
75
+ 'Draft only — catalog feeds still need directory.topics and title/url; ' \
76
+ 'enhance defaults from admission evidence (false when chrome drops are high). ' \
77
+ 'Full schema options live in resource html2rss://schema.',
78
+ input_schema: Contract::CAPTURE_INPUT_SCHEMA,
79
+ handler: :capture_outcome
80
+ },
81
+ {
82
+ name: 'validate',
83
+ kind: :config_xor,
84
+ description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
85
+ 'Call before test. Failures return isError with payload.errors. ' \
86
+ 'Full schema lives in resource html2rss://schema.',
87
+ input_schema: Contract::CONFIG_XOR_SCHEMA,
88
+ annotations: Contract::ANNOTATIONS_VALIDATE,
89
+ call: lambda { |config: nil, yaml: nil, **|
90
+ validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
91
+ Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
92
+ }
93
+ },
94
+ {
95
+ name: 'test',
96
+ kind: :config_xor,
97
+ description: 'Validate schema and execute live extraction (asserting >= min_items items). ' \
98
+ 'Call after capture or validate; on success next_step is apply. ' \
99
+ 'Returns test summary in payload with sample items, timing, failure_kind, ' \
100
+ 'and quality_report (warnings for duplicate URLs, junk titles, native feed). ' \
101
+ 'Set strict_quality to fail on duplicate URLs, >50% junk titles, or short titles.',
102
+ input_schema: Contract::TEST_INPUT_SCHEMA,
103
+ call: lambda { |config: nil, yaml: nil, min_items: 1, strict_quality: false,
104
+ compare_enhance: false, **kwargs|
105
+ feed_config = ConfigArgument.parse(config:, yaml:).config
106
+ test_args = { min_items:, strict_quality:, compare_enhance: }
107
+ test_args[:strategy] = Runtime.coerce_strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
108
+ test_result = Html2rss.test(feed_config, **test_args)
109
+ Outcome.test(test_result)
110
+ }
111
+ },
112
+ {
113
+ name: 'apply',
114
+ kind: :config_xor,
115
+ description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
116
+ 'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
117
+ 'Use after test succeeds.',
118
+ input_schema: Contract::APPLY_INPUT_SCHEMA,
119
+ handler: :apply_outcome
120
+ }
121
+ ].freeze
122
+
123
+ class << self # rubocop:disable Metrics/ClassLength -- registration engine + handlers
124
+ ##
125
+ # Registers all MCP tools on +server+ via +registrar+ (Server.define_envelope_tool).
126
+ #
127
+ # @param server [::MCP::Server]
128
+ # @param registrar [Proc]
129
+ # @return [void]
130
+ def register_all(server, registrar:)
131
+ TOOLS.each { |entry| register_tool(server, registrar, entry) }
132
+ end
133
+
134
+ private
135
+
136
+ def register_tool(server, registrar, entry)
137
+ case entry[:kind]
138
+ when :url then register_url_tool(server, registrar, entry)
139
+ when :batch then register_batch_tool(server, registrar, entry)
140
+ when :config_xor then register_config_xor_tool(server, registrar, entry)
141
+ when :capture then register_capture_tool(server, registrar, entry)
142
+ else raise ArgumentError, "unknown tool kind: #{entry[:kind].inspect}"
143
+ end
144
+ end
145
+
146
+ def register_url_tool(server, registrar, entry)
147
+ handler = entry[:handler] ? method(entry[:handler]) : entry[:call]
148
+ registrar.call(
149
+ server,
150
+ name: entry[:name],
151
+ description: entry[:description],
152
+ input_schema: entry[:input_schema],
153
+ annotations: entry.fetch(:annotations, Contract::ANNOTATIONS_OPEN_WORLD)
154
+ ) do |**kwargs|
155
+ handler.call(**tool_kwargs(kwargs))
156
+ end
157
+ end
158
+
159
+ def register_batch_tool(server, registrar, entry) # rubocop:disable Metrics/MethodLength
160
+ name = entry[:name]
161
+ batch_method = entry[:batch_method]
162
+ limit_default = entry[:limit_default]
163
+ registrar.call(
164
+ server,
165
+ name:,
166
+ description: entry[:description],
167
+ input_schema: entry[:input_schema]
168
+ ) do |urls:, strategy: 'auto', **kwargs|
169
+ concurrency = kwargs.fetch(:concurrency, Batch::DEFAULT_CONCURRENCY)
170
+ batch_args = { urls:, strategy: Runtime.coerce_strategy(strategy), concurrency: }
171
+ batch_args[:limit] = kwargs.fetch(:limit, limit_default) unless limit_default.nil?
172
+ Outcome.public_send(name, Batch.public_send(batch_method, **batch_args))
173
+ end
174
+ end
175
+
176
+ def register_config_xor_tool(server, registrar, entry)
177
+ handler = entry[:handler] ? method(entry[:handler]) : entry[:call]
178
+ registrar.call(
179
+ server,
180
+ name: entry[:name],
181
+ description: entry[:description],
182
+ input_schema: entry[:input_schema],
183
+ annotations: entry.fetch(:annotations, Contract::ANNOTATIONS_OPEN_WORLD)
184
+ ) do |**kwargs|
185
+ handler.call(**tool_kwargs(kwargs))
186
+ end
187
+ end
188
+
189
+ def register_capture_tool(server, registrar, entry)
190
+ handler = method(entry[:handler])
191
+ registrar.call(
192
+ server,
193
+ name: entry[:name],
194
+ description: entry[:description],
195
+ input_schema: entry[:input_schema]
196
+ ) do |**kwargs|
197
+ handler.call(**tool_kwargs(kwargs))
198
+ end
199
+ end
200
+
201
+ def tool_kwargs(inputs)
202
+ args = inputs.dup
203
+ args.delete(:server_context)
204
+ args
205
+ end
206
+
207
+ def scrape_outcome(url:, strategy: 'auto', limit: 25, items_selector: nil)
208
+ wire = Batch.scrape_wire(url:, strategy: Runtime.coerce_strategy(strategy), limit:, items_selector:)
209
+ Outcome.scrape(
210
+ items: wire[:items],
211
+ requested_strategy: wire[:strategy],
212
+ channel_title: wire[:channel_title],
213
+ admission_drops: wire[:admission_drops],
214
+ botasaurus_configured: Runtime.botasaurus_configured?
215
+ )
216
+ end
217
+
218
+ def capture_outcome(url:, strategy: 'auto', items_selector: nil, force: false, topics: nil, title: nil, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
219
+ summary: nil, enhance: nil, limit: nil, max_redirects: nil, max_requests: nil)
220
+ plan = Runtime.coerce_strategy(strategy)
221
+ result = Html2rss::Capture.build(
222
+ url,
223
+ strategy: plan,
224
+ items_selector:,
225
+ force:,
226
+ topics:,
227
+ title:,
228
+ summary:,
229
+ enhance:,
230
+ limit:,
231
+ max_redirects:,
232
+ max_requests:
233
+ )
234
+ Outcome.capture(
235
+ yaml: result.yaml,
236
+ articles_count: result.articles_count,
237
+ has_selectors: result.has_selectors,
238
+ channel_title: result.channel_title,
239
+ requested_strategy: plan,
240
+ segment_strategy: result.segment_strategy,
241
+ selected_strategy: result.selected_strategy,
242
+ admission_drops: result.admission_drops,
243
+ native_feed: result.native_feed,
244
+ suggested_channel_url: result.suggested_channel_url
245
+ )
246
+ end
247
+
248
+ def apply_outcome(url:, config: nil, yaml: nil)
249
+ feed_config = HashUtil.deep_dup(ConfigArgument.parse(config:, yaml:).config)
250
+ feed_config[:channel] ||= {}
251
+ feed_config[:channel][:url] ||= url
252
+ outcome, feed_result = FeedPipeline.new(feed_config).to_outcome_and_result
253
+ apply_feed_outcome(feed_config, feed_result, outcome)
254
+ end
255
+
256
+ def apply_feed_outcome(feed_config, feed_result, outcome) # rubocop:disable Metrics/MethodLength -- quality_report + RSS payload
257
+ rss = feed_result.to_rss
258
+ quality_report = Html2rss::Test.quality_report_for(
259
+ rss.items,
260
+ channel_url: feed_config.dig(:channel, :url).to_s,
261
+ raw_config: feed_config,
262
+ feed_result:,
263
+ pipeline_outcome: outcome,
264
+ probe_native_feed: false
265
+ )
266
+ Outcome.apply(
267
+ rss: rss.to_s,
268
+ item_count: rss.items.size,
269
+ empty: feed_result.empty?,
270
+ quality_report: quality_report.to_h
271
+ )
272
+ end
273
+ end
274
+ end
275
+ end
276
+ end
277
+ end
@@ -20,6 +20,68 @@ module Html2rss
20
20
  # Loopback bind for HTTP transport (local use only).
21
21
  HTTP_BIND_HOST = '127.0.0.1'
22
22
 
23
+ # Declarative MCP resource registrations consumed by {register_resources}.
24
+ RESOURCES = [
25
+ {
26
+ uri: 'html2rss://schema',
27
+ name: 'Configuration JSON Schema',
28
+ description: 'Full JSON Schema for html2rss feed configurations',
29
+ mime_type: 'application/json',
30
+ body: lambda {
31
+ [{ uri: 'html2rss://schema', mimeType: 'application/json',
32
+ text: Html2rss::Config.json_schema_json(pretty: true) }]
33
+ }
34
+ },
35
+ {
36
+ uri: 'html2rss://extractors',
37
+ name: 'Available Extractors',
38
+ description: 'Registered extractor names for selector configs ' \
39
+ '(full option docs live in html2rss://schema $defs)',
40
+ mime_type: 'application/json',
41
+ body: lambda {
42
+ extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
43
+ [{ uri: 'html2rss://extractors', mimeType: 'application/json',
44
+ text: JSON.pretty_generate(extractors) }]
45
+ }
46
+ },
47
+ {
48
+ uri: 'html2rss://strategies',
49
+ name: 'Available Strategies',
50
+ description: 'Published MCP request strategy names',
51
+ mime_type: 'application/json',
52
+ body: lambda {
53
+ [{ uri: 'html2rss://strategies', mimeType: 'application/json',
54
+ text: JSON.generate(Contract::STRATEGIES) }]
55
+ }
56
+ },
57
+ {
58
+ uri: 'html2rss://runtime',
59
+ name: 'Runtime capabilities',
60
+ description: 'Gem version, MCP contract version, catalog fingerprint, tool names, and Botasaurus config',
61
+ mime_type: 'application/json',
62
+ body: lambda {
63
+ [{ uri: 'html2rss://runtime', mimeType: 'application/json',
64
+ text: JSON.pretty_generate(Runtime.snapshot.to_h) }]
65
+ }
66
+ }
67
+ ].freeze
68
+
69
+ # Declarative MCP prompt registrations; SDK argument objects built at {register_prompts} time.
70
+ PROMPTS = [
71
+ {
72
+ name: 'scrape-webpage',
73
+ description: 'Guided one-shot scrape: one scrape call (auto already falls back)',
74
+ arguments: [{ name: 'url', description: 'URL to scrape', required: true }],
75
+ body: ->(args) { Outcome::Playbook.scrape_webpage_prompt(args.fetch(:url)) }
76
+ },
77
+ {
78
+ name: 'capture-feed-config',
79
+ description: 'Guided capture → test → apply; YAML draft plus catalog rewrite',
80
+ arguments: [{ name: 'url', description: 'URL to analyze', required: true }],
81
+ body: ->(args) { Outcome::Playbook.capture_feed_config_prompt(args.fetch(:url)) }
82
+ }
83
+ ].freeze
84
+
23
85
  class << self # rubocop:disable Metrics/ClassLength
24
86
  ##
25
87
  # Starts the MCP server with the given transport.
@@ -140,28 +202,8 @@ module Html2rss
140
202
  "(#{error.message}). Install them or use --transport stdio."
141
203
  end
142
204
 
143
- def instructions_text # rubocop:disable Metrics/MethodLength -- agent decision tree is the published contract
144
- <<~TEXT.strip
145
- html2rss MCP — decide which tool to call:
146
-
147
- 1. Need articles now (no saved config)? → scrape_url (1 call)
148
- - strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
149
- - Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
150
- 2. Need a reusable feed YAML? → capture_config → validate_config → apply_config
151
- - capture_config returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Strive enhance: true (false only when chrome leaks).
152
- - validate_config / apply_config accept config hash XOR yaml string.
153
- - apply_config isError when zero RSS items (ship gate). Confirm payload.item_count.
154
- 3. Weak scrape/capture or recon (final URL, status, https→http, rel=alternate feeds)? → inspect_url only if weak or recon.
155
- 4. Have a config already? → validate_config (must succeed) → apply_config
156
- 5. Schema / extractors / strategies / runtime → resources html2rss://schema|extractors|strategies|runtime
157
-
158
- Prefer capture_config for durable config; scrape_url for one-shot extraction.
159
- Follow envelope next_step and guidance. Botasaurus needs BOTASAURUS_SCRAPER_URL in this process env (boolean at html2rss://runtime; the URL is never returned).
160
- TEXT
161
- end
162
-
163
- def botasaurus_configured?
164
- !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
205
+ def instructions_text
206
+ Outcome::Playbook.instructions
165
207
  end
166
208
 
167
209
  def tool_error_response(error)
@@ -175,7 +217,7 @@ module Html2rss
175
217
  tool_error_response(error)
176
218
  end
177
219
 
178
- # rubocop:disable Metrics/MethodLength -- listing fields stay together
220
+ # rubocop:disable-next Metrics/MethodLength -- listing fields stay together
179
221
  def define_envelope_tool(server, name:, description:, input_schema:,
180
222
  annotations: Contract::ANNOTATIONS_OPEN_WORLD)
181
223
  run = method(:handle_tool_call)
@@ -190,201 +232,38 @@ module Html2rss
190
232
  run.call { yield(**kwargs) }
191
233
  end
192
234
  end
193
- # rubocop:enable Metrics/MethodLength
194
235
 
195
236
  def register_tools(server)
196
- register_scrape_url(server)
197
- register_inspect_url(server)
198
- register_capture_config(server)
199
- register_validate_config(server)
200
- register_apply_config(server)
201
- end
202
-
203
- def register_scrape_url(server)
204
- define_envelope_tool(
205
- server,
206
- name: 'scrape_url',
207
- description: 'One-shot article extraction as JSON Feed items. ' \
208
- 'Use when you need articles now without a saved config. ' \
209
- 'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
210
- input_schema: Contract::SCRAPE_INPUT_SCHEMA
211
- ) do |server_context:, url:, strategy: 'auto', limit: 25, items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
212
- scrape_outcome(url:, strategy:, limit:, items_selector:)
213
- end
214
- end
215
-
216
- def scrape_outcome(url:, strategy:, limit:, items_selector:)
217
- plan = (strategy || :auto).to_sym
218
- feed_result = Html2rss.auto_feed_result(url, strategy: plan, limit:, items_selector:)
219
- feed = feed_result.to_json_feed
220
- Outcome.scrape(
221
- items: feed[:items] || [],
222
- requested_strategy: plan,
223
- channel_title: feed[:title],
224
- admission_drops: feed_result.status.admission_drops,
225
- botasaurus_configured: botasaurus_configured?
226
- )
227
- end
228
-
229
- def register_inspect_url(server)
230
- define_envelope_tool(
231
- server,
232
- name: 'inspect_url',
233
- description: 'Diagnostic page analysis (scrapers, SST, segments) plus recon: ' \
234
- 'final_url, status, scheme_downgrade, rel=alternate RSS/Atom feeds. ' \
235
- 'Use when scrape/capture is weak or you need those recon facts.',
236
- input_schema: Contract::INSPECT_INPUT_SCHEMA
237
- ) do |server_context:, url:, strategy: 'auto'| # rubocop:disable Lint/UnusedBlockArgument
238
- Outcome.inspect(payload: Inspect.call(url:, strategy:))
239
- end
240
- end
241
-
242
- def register_capture_config(server) # rubocop:disable Metrics/MethodLength
243
- define_envelope_tool(
244
- server,
245
- name: 'capture_config',
246
- description: 'Derive a reusable html2rss feed config from a URL. ' \
247
- 'Use when the goal is a durable YAML (then validate_config). ' \
248
- 'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
249
- 'Draft only — catalog feeds still need directory.topics and title/url; ' \
250
- 'strive enhance: true. Full schema options live in resource html2rss://schema.',
251
- input_schema: Contract::CAPTURE_INPUT_SCHEMA
252
- ) do |server_context:, url:, strategy: 'auto', items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
253
- capture_outcome(url:, strategy:, items_selector:)
254
- end
255
- end
256
-
257
- def capture_outcome(url:, strategy:, items_selector:) # rubocop:disable Metrics/MethodLength -- CaptureResult maps 1:1 onto Outcome
258
- plan = (strategy || :auto).to_sym
259
- result = Html2rss::Capture.build(url, strategy: plan, items_selector:)
260
- Outcome.capture(
261
- yaml: Config.to_yaml(result.config),
262
- articles_count: result.articles_count,
263
- has_selectors: result.has_selectors,
264
- channel_title: result.channel_title,
265
- requested_strategy: plan,
266
- segment_strategy: result.segment_strategy,
267
- selected_strategy: result.selected_strategy,
268
- admission_drops: result.admission_drops
269
- )
270
- end
271
-
272
- def register_validate_config(server) # rubocop:disable Metrics/MethodLength -- description is the published contract
273
- define_envelope_tool(
274
- server,
275
- name: 'validate_config',
276
- description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
277
- 'Call before apply_config. Failures return isError with payload.errors. ' \
278
- 'Full schema lives in resource html2rss://schema.',
279
- input_schema: Contract::CONFIG_XOR_SCHEMA,
280
- annotations: Contract::ANNOTATIONS_VALIDATE
281
- ) do |server_context:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
282
- validate_outcome(config:, yaml:)
283
- end
284
- end
285
-
286
- def validate_outcome(config:, yaml:)
287
- validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
288
- Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
289
- end
290
-
291
- def register_apply_config(server)
292
- define_envelope_tool(
293
- server,
294
- name: 'apply_config',
295
- description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
296
- 'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
297
- 'Use after validate_config succeeds.',
298
- input_schema: Contract::APPLY_INPUT_SCHEMA
299
- ) do |server_context:, url:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
300
- apply_outcome(url:, config:, yaml:)
237
+ Tools.register_all(server, registrar: method(:define_envelope_tool))
238
+ end
239
+
240
+ def register_resources(server)
241
+ RESOURCES.each do |entry|
242
+ server.define_resource(
243
+ uri: entry[:uri],
244
+ name: entry[:name],
245
+ description: entry[:description],
246
+ mime_type: entry[:mime_type]
247
+ ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
248
+ entry[:body].call
249
+ end
301
250
  end
302
251
  end
303
252
 
304
- def apply_outcome(url:, config:, yaml:)
305
- feed_config = ConfigArgument.parse(config:, yaml:).config
306
- feed_config[:channel] ||= {}
307
- feed_config[:channel][:url] ||= url
308
- feed_result = Html2rss.feed_result(feed_config)
309
- rss = feed_result.to_rss
310
- Outcome.apply(rss: rss.to_s, item_count: rss.items.size, empty: feed_result.empty?)
311
- end
312
-
313
- def register_resources(server) # rubocop:disable Metrics/MethodLength
314
- configured = method(:botasaurus_configured?)
315
- server.define_resource(
316
- uri: 'html2rss://schema',
317
- name: 'Configuration JSON Schema',
318
- description: 'Full JSON Schema for html2rss feed configurations',
319
- mime_type: 'application/json'
320
- ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
321
- schema = Html2rss::Config.json_schema_json(pretty: true)
322
- [{ uri: 'html2rss://schema', mimeType: 'application/json', text: schema }]
323
- end
324
-
325
- server.define_resource(
326
- uri: 'html2rss://extractors',
327
- name: 'Available Extractors',
328
- description: 'Registered extractor names for selector configs ' \
329
- '(full option docs live in html2rss://schema $defs)',
330
- mime_type: 'application/json'
331
- ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
332
- extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
333
- [{ uri: 'html2rss://extractors', mimeType: 'application/json',
334
- text: JSON.pretty_generate(extractors) }]
335
- end
336
-
337
- server.define_resource(
338
- uri: 'html2rss://strategies',
339
- name: 'Available Strategies',
340
- description: 'Published MCP request strategy names',
341
- mime_type: 'application/json'
342
- ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
343
- [{ uri: 'html2rss://strategies', mimeType: 'application/json',
344
- text: JSON.generate(Contract::STRATEGIES) }]
345
- end
346
-
347
- server.define_resource(
348
- uri: 'html2rss://runtime',
349
- name: 'Runtime capabilities',
350
- description: 'Whether optional transports are configured in this MCP process (never leaks secrets)',
351
- mime_type: 'application/json'
352
- ) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
353
- [{ uri: 'html2rss://runtime', mimeType: 'application/json',
354
- text: JSON.pretty_generate(botasaurus_configured: configured.call) }]
355
- end
356
- end
357
-
358
- def register_prompts(server)
359
- register_scrape_webpage_prompt(server)
360
- register_capture_feed_config_prompt(server)
361
- end
362
-
363
- def register_scrape_webpage_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
253
+ def register_prompts(server) # rubocop:disable Metrics/MethodLength -- prompt argument mapping
364
254
  to_result = method(:prompt_result)
365
- text_for = method(:scrape_webpage_text)
366
- server.define_prompt(
367
- name: 'scrape-webpage',
368
- description: 'Guided one-shot scrape: one scrape_url call (auto already falls back)',
369
- arguments: [
370
- ::MCP::Prompt::Argument.new(name: 'url', description: 'URL to scrape', required: true)
371
- ]
372
- ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
373
- to_result.call(text_for.call(args.fetch(:url)))
374
- end
375
- end
376
-
377
- def register_capture_feed_config_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
378
- to_result = method(:prompt_result)
379
- text_for = method(:capture_feed_config_text)
380
- server.define_prompt(
381
- name: 'capture-feed-config',
382
- description: 'Guided capture → validate → apply; YAML draft plus catalog rewrite',
383
- arguments: [
384
- ::MCP::Prompt::Argument.new(name: 'url', description: 'URL to analyze', required: true)
385
- ]
386
- ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
387
- to_result.call(text_for.call(args.fetch(:url)))
255
+ PROMPTS.each do |entry|
256
+ arguments = entry[:arguments].map do |spec|
257
+ ::MCP::Prompt::Argument.new(name: spec[:name], description: spec[:description],
258
+ required: spec.fetch(:required, false))
259
+ end
260
+ server.define_prompt(
261
+ name: entry[:name],
262
+ description: entry[:description],
263
+ arguments:
264
+ ) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
265
+ to_result.call(entry[:body].call(args))
266
+ end
388
267
  end
389
268
  end
390
269
 
@@ -395,26 +274,6 @@ module Html2rss
395
274
  ]
396
275
  )
397
276
  end
398
-
399
- def scrape_webpage_text(url)
400
- <<~MSG.strip
401
- Scrape #{url} with scrape_url (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
402
- Follow envelope next_step and guidance. Call inspect_url only if articles are empty/weak or you need recon (final_url, status, scheme_downgrade, alternate_feeds).
403
- Do not retry scrape_url with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
404
- Return payload.items (not a raw JSON array).
405
- MSG
406
- end
407
-
408
- def capture_feed_config_text(url)
409
- <<~MSG.strip
410
- Build a reusable html2rss feed config for #{url}:
411
- 1) capture_config — YAML is payload.yaml. Check payload.articles_count and payload.has_selectors. Strive to keep enhance: true (false only when chrome leaks into items).
412
- 2) Follow next_step. If weak or you need recon, inspect_url. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
413
- 3) validate_config with yaml (or config hash) — must not be isError
414
- 4) apply_config — isError if zero items. Confirm payload.item_count before shipping.
415
- If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
416
- MSG
417
- end
418
277
  end
419
278
  end
420
279
  end
data/lib/html2rss/mcp.rb CHANGED
@@ -3,6 +3,9 @@
3
3
  module Html2rss
4
4
  ##
5
5
  # MCP server for AI client consumption.
6
+ #
7
+ # {include:file:lib/html2rss/mcp/README.md}
8
+ #
6
9
  # Lazy-loads the mcp gem; no cost when the server is not started.
7
10
  module MCP
8
11
  class << self