html2rss 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +82 -25
- data/html2rss.gemspec +0 -1
- data/lib/html2rss/article.rb +1 -2
- data/lib/html2rss/auto_source/README.md +8 -8
- data/lib/html2rss/auto_source/cleanup.rb +92 -1
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
- data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
- data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
- data/lib/html2rss/auto_source/scraper.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
- data/lib/html2rss/auto_source/segmenter.rb +1 -2
- data/lib/html2rss/auto_source.rb +25 -11
- data/lib/html2rss/batch.rb +169 -0
- data/lib/html2rss/capture/README.md +15 -6
- data/lib/html2rss/capture.rb +158 -15
- data/lib/html2rss/channel.rb +1 -2
- data/lib/html2rss/cli/probe_view.rb +66 -0
- data/lib/html2rss/cli/render.rb +124 -0
- data/lib/html2rss/cli/validate.rb +91 -0
- data/lib/html2rss/cli.rb +318 -193
- data/lib/html2rss/config/request_headers.rb +4 -14
- data/lib/html2rss/config/schema.rb +7 -5
- data/lib/html2rss/config/validator.rb +1 -0
- data/lib/html2rss/config.rb +59 -22
- data/lib/html2rss/doctor/botasaurus.rb +130 -0
- data/lib/html2rss/feed_builder/rss.rb +1 -2
- data/lib/html2rss/feed_pipeline/README.md +5 -5
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
- data/lib/html2rss/feed_pipeline.rb +27 -16
- data/lib/html2rss/feed_resolution/README.md +11 -11
- data/lib/html2rss/feed_resolution.rb +6 -12
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
- data/lib/html2rss/html/article_extractor.rb +4 -9
- data/lib/html2rss/html/feed_link.rb +4 -2
- data/lib/html2rss/html/navigator.rb +4 -39
- data/lib/html2rss/html/probe.rb +82 -0
- data/lib/html2rss/html/sst_article_extractor.rb +1 -2
- data/lib/html2rss/link_destination/noise_policy.rb +1 -2
- data/lib/html2rss/link_destination/path_classifier.rb +20 -19
- data/lib/html2rss/mcp/README.md +80 -0
- data/lib/html2rss/mcp/contract.rb +135 -13
- data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
- data/lib/html2rss/mcp/outcome.rb +112 -42
- data/lib/html2rss/mcp/runtime.rb +45 -0
- data/lib/html2rss/mcp/server/tools.rb +277 -0
- data/lib/html2rss/mcp/server.rb +91 -232
- data/lib/html2rss/mcp.rb +3 -0
- data/lib/html2rss/page_recon/README.md +69 -0
- data/lib/html2rss/page_recon/diagnostics.rb +211 -0
- data/lib/html2rss/page_recon.rb +56 -2
- data/lib/html2rss/recon.rb +252 -0
- data/lib/html2rss/request_service/blocked_surface.rb +1 -0
- data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
- data/lib/html2rss/request_service/policy.rb +1 -2
- data/lib/html2rss/request_service/response.rb +3 -4
- data/lib/html2rss/request_service/strategy.rb +1 -2
- data/lib/html2rss/request_service.rb +2 -0
- data/lib/html2rss/request_session.rb +1 -2
- data/lib/html2rss/scrape_target.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
- data/lib/html2rss/selectors/post_processors.rb +0 -1
- data/lib/html2rss/selectors.rb +33 -10
- data/lib/html2rss/sst/attrs.rb +1 -2
- data/lib/html2rss/sst/normalizer.rb +49 -11
- data/lib/html2rss/status.rb +3 -6
- data/lib/html2rss/syndication/README.md +6 -6
- data/lib/html2rss/syndication/discovery.rb +2 -4
- data/lib/html2rss/test/enhance_audit.rb +192 -0
- data/lib/html2rss/test.rb +458 -0
- data/lib/html2rss/url.rb +1 -2
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +157 -35
- data/schema/html2rss-config.schema.json +9 -29
- metadata +17 -18
- data/lib/html2rss/mcp/inspect.rb +0 -138
- data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
module MCP
|
|
5
|
+
module Server
|
|
6
|
+
##
|
|
7
|
+
# MCP tool registration and outcome mapping over public html2rss APIs.
|
|
8
|
+
module Tools # rubocop:disable Metrics/ModuleLength -- declarative registry + substantive handlers
|
|
9
|
+
# Declarative MCP tool registrations consumed by {register_all}.
|
|
10
|
+
TOOLS = [
|
|
11
|
+
{
|
|
12
|
+
name: 'scrape',
|
|
13
|
+
kind: :url,
|
|
14
|
+
description: 'One-shot article extraction as JSON Feed items. ' \
|
|
15
|
+
'Use when you need articles now without a saved config. ' \
|
|
16
|
+
'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
|
|
17
|
+
input_schema: Contract::SCRAPE_INPUT_SCHEMA,
|
|
18
|
+
handler: :scrape_outcome
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
name: 'inspect',
|
|
22
|
+
kind: :url,
|
|
23
|
+
description: 'Diagnostic page analysis (scrapers, SST, segments, final URL, status, ' \
|
|
24
|
+
'rel=alternate feeds). Use recon for BUILD/DEFER/DROP verdict and native_feed preference.',
|
|
25
|
+
input_schema: Contract::INSPECT_INPUT_SCHEMA,
|
|
26
|
+
call: lambda { |url:, strategy: 'auto', **|
|
|
27
|
+
Outcome.inspect(
|
|
28
|
+
report: PageRecon::Diagnostics.call(
|
|
29
|
+
url:, strategy: Runtime.coerce_strategy(strategy), deep: false
|
|
30
|
+
)
|
|
31
|
+
)
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
name: 'recon',
|
|
36
|
+
kind: :url,
|
|
37
|
+
description: 'Curation verdict and native_feed preference for a URL. ' \
|
|
38
|
+
'Use after inspect when alternates warrant deeper recon, or when you need BUILD/DEFER/DROP.',
|
|
39
|
+
input_schema: Contract::RECON_INPUT_SCHEMA,
|
|
40
|
+
call: lambda { |url:, strategy: 'auto', **|
|
|
41
|
+
Outcome.recon(result: Html2rss.recon(url, strategy: Runtime.coerce_strategy(strategy)))
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
name: 'batch_scrape',
|
|
46
|
+
kind: :batch,
|
|
47
|
+
batch_method: :batch_scrape,
|
|
48
|
+
limit_default: 10,
|
|
49
|
+
description: 'Scrape multiple URLs in parallel with per-URL error isolation. ' \
|
|
50
|
+
'Returns structured JSON Feed items and extraction counts.',
|
|
51
|
+
input_schema: Contract::BATCH_SCRAPE_INPUT_SCHEMA
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
name: 'batch_inspect',
|
|
55
|
+
kind: :batch,
|
|
56
|
+
batch_method: :batch_inspect,
|
|
57
|
+
description: 'Inspect multiple URLs in parallel with per-URL error isolation. ' \
|
|
58
|
+
'Returns final redirected URLs, status codes, and rel="alternate" feeds.',
|
|
59
|
+
input_schema: Contract::BATCH_INSPECT_INPUT_SCHEMA
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
name: 'batch_recon',
|
|
63
|
+
kind: :batch,
|
|
64
|
+
batch_method: :batch_recon,
|
|
65
|
+
description: 'Run recon across multiple URLs in parallel with per-URL error isolation. ' \
|
|
66
|
+
'Returns verdict, native_feed, and surface classification per URL.',
|
|
67
|
+
input_schema: Contract::BATCH_RECON_INPUT_SCHEMA
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'capture',
|
|
71
|
+
kind: :capture,
|
|
72
|
+
description: 'Derive a reusable html2rss feed config from a URL. ' \
|
|
73
|
+
'Use when the goal is a durable YAML (then test → apply). ' \
|
|
74
|
+
'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
|
|
75
|
+
'Draft only — catalog feeds still need directory.topics and title/url; ' \
|
|
76
|
+
'enhance defaults from admission evidence (false when chrome drops are high). ' \
|
|
77
|
+
'Full schema options live in resource html2rss://schema.',
|
|
78
|
+
input_schema: Contract::CAPTURE_INPUT_SCHEMA,
|
|
79
|
+
handler: :capture_outcome
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
name: 'validate',
|
|
83
|
+
kind: :config_xor,
|
|
84
|
+
description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
|
|
85
|
+
'Call before test. Failures return isError with payload.errors. ' \
|
|
86
|
+
'Full schema lives in resource html2rss://schema.',
|
|
87
|
+
input_schema: Contract::CONFIG_XOR_SCHEMA,
|
|
88
|
+
annotations: Contract::ANNOTATIONS_VALIDATE,
|
|
89
|
+
call: lambda { |config: nil, yaml: nil, **|
|
|
90
|
+
validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
|
|
91
|
+
Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
name: 'test',
|
|
96
|
+
kind: :config_xor,
|
|
97
|
+
description: 'Validate schema and execute live extraction (asserting >= min_items items). ' \
|
|
98
|
+
'Call after capture or validate; on success next_step is apply. ' \
|
|
99
|
+
'Returns test summary in payload with sample items, timing, failure_kind, ' \
|
|
100
|
+
'and quality_report (warnings for duplicate URLs, junk titles, native feed). ' \
|
|
101
|
+
'Set strict_quality to fail on duplicate URLs, >50% junk titles, or short titles.',
|
|
102
|
+
input_schema: Contract::TEST_INPUT_SCHEMA,
|
|
103
|
+
call: lambda { |config: nil, yaml: nil, min_items: 1, strict_quality: false,
|
|
104
|
+
compare_enhance: false, **kwargs|
|
|
105
|
+
feed_config = ConfigArgument.parse(config:, yaml:).config
|
|
106
|
+
test_args = { min_items:, strict_quality:, compare_enhance: }
|
|
107
|
+
test_args[:strategy] = Runtime.coerce_strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
|
|
108
|
+
test_result = Html2rss.test(feed_config, **test_args)
|
|
109
|
+
Outcome.test(test_result)
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
name: 'apply',
|
|
114
|
+
kind: :config_xor,
|
|
115
|
+
description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
|
|
116
|
+
'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
|
|
117
|
+
'Use after test succeeds.',
|
|
118
|
+
input_schema: Contract::APPLY_INPUT_SCHEMA,
|
|
119
|
+
handler: :apply_outcome
|
|
120
|
+
}
|
|
121
|
+
].freeze
|
|
122
|
+
|
|
123
|
+
class << self # rubocop:disable Metrics/ClassLength -- registration engine + handlers
|
|
124
|
+
##
|
|
125
|
+
# Registers all MCP tools on +server+ via +registrar+ (Server.define_envelope_tool).
|
|
126
|
+
#
|
|
127
|
+
# @param server [::MCP::Server]
|
|
128
|
+
# @param registrar [Proc]
|
|
129
|
+
# @return [void]
|
|
130
|
+
def register_all(server, registrar:)
|
|
131
|
+
TOOLS.each { |entry| register_tool(server, registrar, entry) }
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
private
|
|
135
|
+
|
|
136
|
+
def register_tool(server, registrar, entry)
|
|
137
|
+
case entry[:kind]
|
|
138
|
+
when :url then register_url_tool(server, registrar, entry)
|
|
139
|
+
when :batch then register_batch_tool(server, registrar, entry)
|
|
140
|
+
when :config_xor then register_config_xor_tool(server, registrar, entry)
|
|
141
|
+
when :capture then register_capture_tool(server, registrar, entry)
|
|
142
|
+
else raise ArgumentError, "unknown tool kind: #{entry[:kind].inspect}"
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def register_url_tool(server, registrar, entry)
|
|
147
|
+
handler = entry[:handler] ? method(entry[:handler]) : entry[:call]
|
|
148
|
+
registrar.call(
|
|
149
|
+
server,
|
|
150
|
+
name: entry[:name],
|
|
151
|
+
description: entry[:description],
|
|
152
|
+
input_schema: entry[:input_schema],
|
|
153
|
+
annotations: entry.fetch(:annotations, Contract::ANNOTATIONS_OPEN_WORLD)
|
|
154
|
+
) do |**kwargs|
|
|
155
|
+
handler.call(**tool_kwargs(kwargs))
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def register_batch_tool(server, registrar, entry) # rubocop:disable Metrics/MethodLength
|
|
160
|
+
name = entry[:name]
|
|
161
|
+
batch_method = entry[:batch_method]
|
|
162
|
+
limit_default = entry[:limit_default]
|
|
163
|
+
registrar.call(
|
|
164
|
+
server,
|
|
165
|
+
name:,
|
|
166
|
+
description: entry[:description],
|
|
167
|
+
input_schema: entry[:input_schema]
|
|
168
|
+
) do |urls:, strategy: 'auto', **kwargs|
|
|
169
|
+
concurrency = kwargs.fetch(:concurrency, Batch::DEFAULT_CONCURRENCY)
|
|
170
|
+
batch_args = { urls:, strategy: Runtime.coerce_strategy(strategy), concurrency: }
|
|
171
|
+
batch_args[:limit] = kwargs.fetch(:limit, limit_default) unless limit_default.nil?
|
|
172
|
+
Outcome.public_send(name, Batch.public_send(batch_method, **batch_args))
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def register_config_xor_tool(server, registrar, entry)
|
|
177
|
+
handler = entry[:handler] ? method(entry[:handler]) : entry[:call]
|
|
178
|
+
registrar.call(
|
|
179
|
+
server,
|
|
180
|
+
name: entry[:name],
|
|
181
|
+
description: entry[:description],
|
|
182
|
+
input_schema: entry[:input_schema],
|
|
183
|
+
annotations: entry.fetch(:annotations, Contract::ANNOTATIONS_OPEN_WORLD)
|
|
184
|
+
) do |**kwargs|
|
|
185
|
+
handler.call(**tool_kwargs(kwargs))
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def register_capture_tool(server, registrar, entry)
|
|
190
|
+
handler = method(entry[:handler])
|
|
191
|
+
registrar.call(
|
|
192
|
+
server,
|
|
193
|
+
name: entry[:name],
|
|
194
|
+
description: entry[:description],
|
|
195
|
+
input_schema: entry[:input_schema]
|
|
196
|
+
) do |**kwargs|
|
|
197
|
+
handler.call(**tool_kwargs(kwargs))
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def tool_kwargs(inputs)
|
|
202
|
+
args = inputs.dup
|
|
203
|
+
args.delete(:server_context)
|
|
204
|
+
args
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def scrape_outcome(url:, strategy: 'auto', limit: 25, items_selector: nil)
|
|
208
|
+
wire = Batch.scrape_wire(url:, strategy: Runtime.coerce_strategy(strategy), limit:, items_selector:)
|
|
209
|
+
Outcome.scrape(
|
|
210
|
+
items: wire[:items],
|
|
211
|
+
requested_strategy: wire[:strategy],
|
|
212
|
+
channel_title: wire[:channel_title],
|
|
213
|
+
admission_drops: wire[:admission_drops],
|
|
214
|
+
botasaurus_configured: Runtime.botasaurus_configured?
|
|
215
|
+
)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def capture_outcome(url:, strategy: 'auto', items_selector: nil, force: false, topics: nil, title: nil, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
|
|
219
|
+
summary: nil, enhance: nil, limit: nil, max_redirects: nil, max_requests: nil)
|
|
220
|
+
plan = Runtime.coerce_strategy(strategy)
|
|
221
|
+
result = Html2rss::Capture.build(
|
|
222
|
+
url,
|
|
223
|
+
strategy: plan,
|
|
224
|
+
items_selector:,
|
|
225
|
+
force:,
|
|
226
|
+
topics:,
|
|
227
|
+
title:,
|
|
228
|
+
summary:,
|
|
229
|
+
enhance:,
|
|
230
|
+
limit:,
|
|
231
|
+
max_redirects:,
|
|
232
|
+
max_requests:
|
|
233
|
+
)
|
|
234
|
+
Outcome.capture(
|
|
235
|
+
yaml: result.yaml,
|
|
236
|
+
articles_count: result.articles_count,
|
|
237
|
+
has_selectors: result.has_selectors,
|
|
238
|
+
channel_title: result.channel_title,
|
|
239
|
+
requested_strategy: plan,
|
|
240
|
+
segment_strategy: result.segment_strategy,
|
|
241
|
+
selected_strategy: result.selected_strategy,
|
|
242
|
+
admission_drops: result.admission_drops,
|
|
243
|
+
native_feed: result.native_feed,
|
|
244
|
+
suggested_channel_url: result.suggested_channel_url
|
|
245
|
+
)
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def apply_outcome(url:, config: nil, yaml: nil)
|
|
249
|
+
feed_config = HashUtil.deep_dup(ConfigArgument.parse(config:, yaml:).config)
|
|
250
|
+
feed_config[:channel] ||= {}
|
|
251
|
+
feed_config[:channel][:url] ||= url
|
|
252
|
+
outcome, feed_result = FeedPipeline.new(feed_config).to_outcome_and_result
|
|
253
|
+
apply_feed_outcome(feed_config, feed_result, outcome)
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def apply_feed_outcome(feed_config, feed_result, outcome) # rubocop:disable Metrics/MethodLength -- quality_report + RSS payload
|
|
257
|
+
rss = feed_result.to_rss
|
|
258
|
+
quality_report = Html2rss::Test.quality_report_for(
|
|
259
|
+
rss.items,
|
|
260
|
+
channel_url: feed_config.dig(:channel, :url).to_s,
|
|
261
|
+
raw_config: feed_config,
|
|
262
|
+
feed_result:,
|
|
263
|
+
pipeline_outcome: outcome,
|
|
264
|
+
probe_native_feed: false
|
|
265
|
+
)
|
|
266
|
+
Outcome.apply(
|
|
267
|
+
rss: rss.to_s,
|
|
268
|
+
item_count: rss.items.size,
|
|
269
|
+
empty: feed_result.empty?,
|
|
270
|
+
quality_report: quality_report.to_h
|
|
271
|
+
)
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
end
|
data/lib/html2rss/mcp/server.rb
CHANGED
|
@@ -20,6 +20,68 @@ module Html2rss
|
|
|
20
20
|
# Loopback bind for HTTP transport (local use only).
|
|
21
21
|
HTTP_BIND_HOST = '127.0.0.1'
|
|
22
22
|
|
|
23
|
+
# Declarative MCP resource registrations consumed by {register_resources}.
|
|
24
|
+
RESOURCES = [
|
|
25
|
+
{
|
|
26
|
+
uri: 'html2rss://schema',
|
|
27
|
+
name: 'Configuration JSON Schema',
|
|
28
|
+
description: 'Full JSON Schema for html2rss feed configurations',
|
|
29
|
+
mime_type: 'application/json',
|
|
30
|
+
body: lambda {
|
|
31
|
+
[{ uri: 'html2rss://schema', mimeType: 'application/json',
|
|
32
|
+
text: Html2rss::Config.json_schema_json(pretty: true) }]
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
uri: 'html2rss://extractors',
|
|
37
|
+
name: 'Available Extractors',
|
|
38
|
+
description: 'Registered extractor names for selector configs ' \
|
|
39
|
+
'(full option docs live in html2rss://schema $defs)',
|
|
40
|
+
mime_type: 'application/json',
|
|
41
|
+
body: lambda {
|
|
42
|
+
extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
|
|
43
|
+
[{ uri: 'html2rss://extractors', mimeType: 'application/json',
|
|
44
|
+
text: JSON.pretty_generate(extractors) }]
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
uri: 'html2rss://strategies',
|
|
49
|
+
name: 'Available Strategies',
|
|
50
|
+
description: 'Published MCP request strategy names',
|
|
51
|
+
mime_type: 'application/json',
|
|
52
|
+
body: lambda {
|
|
53
|
+
[{ uri: 'html2rss://strategies', mimeType: 'application/json',
|
|
54
|
+
text: JSON.generate(Contract::STRATEGIES) }]
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
uri: 'html2rss://runtime',
|
|
59
|
+
name: 'Runtime capabilities',
|
|
60
|
+
description: 'Gem version, MCP contract version, catalog fingerprint, tool names, and Botasaurus config',
|
|
61
|
+
mime_type: 'application/json',
|
|
62
|
+
body: lambda {
|
|
63
|
+
[{ uri: 'html2rss://runtime', mimeType: 'application/json',
|
|
64
|
+
text: JSON.pretty_generate(Runtime.snapshot.to_h) }]
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
].freeze
|
|
68
|
+
|
|
69
|
+
# Declarative MCP prompt registrations; SDK argument objects built at {register_prompts} time.
|
|
70
|
+
PROMPTS = [
|
|
71
|
+
{
|
|
72
|
+
name: 'scrape-webpage',
|
|
73
|
+
description: 'Guided one-shot scrape: one scrape call (auto already falls back)',
|
|
74
|
+
arguments: [{ name: 'url', description: 'URL to scrape', required: true }],
|
|
75
|
+
body: ->(args) { Outcome::Playbook.scrape_webpage_prompt(args.fetch(:url)) }
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
name: 'capture-feed-config',
|
|
79
|
+
description: 'Guided capture → test → apply; YAML draft plus catalog rewrite',
|
|
80
|
+
arguments: [{ name: 'url', description: 'URL to analyze', required: true }],
|
|
81
|
+
body: ->(args) { Outcome::Playbook.capture_feed_config_prompt(args.fetch(:url)) }
|
|
82
|
+
}
|
|
83
|
+
].freeze
|
|
84
|
+
|
|
23
85
|
class << self # rubocop:disable Metrics/ClassLength
|
|
24
86
|
##
|
|
25
87
|
# Starts the MCP server with the given transport.
|
|
@@ -140,28 +202,8 @@ module Html2rss
|
|
|
140
202
|
"(#{error.message}). Install them or use --transport stdio."
|
|
141
203
|
end
|
|
142
204
|
|
|
143
|
-
def instructions_text
|
|
144
|
-
|
|
145
|
-
html2rss MCP — decide which tool to call:
|
|
146
|
-
|
|
147
|
-
1. Need articles now (no saved config)? → scrape_url (1 call)
|
|
148
|
-
- strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
|
|
149
|
-
- Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
|
|
150
|
-
2. Need a reusable feed YAML? → capture_config → validate_config → apply_config
|
|
151
|
-
- capture_config returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Strive enhance: true (false only when chrome leaks).
|
|
152
|
-
- validate_config / apply_config accept config hash XOR yaml string.
|
|
153
|
-
- apply_config isError when zero RSS items (ship gate). Confirm payload.item_count.
|
|
154
|
-
3. Weak scrape/capture or recon (final URL, status, https→http, rel=alternate feeds)? → inspect_url only if weak or recon.
|
|
155
|
-
4. Have a config already? → validate_config (must succeed) → apply_config
|
|
156
|
-
5. Schema / extractors / strategies / runtime → resources html2rss://schema|extractors|strategies|runtime
|
|
157
|
-
|
|
158
|
-
Prefer capture_config for durable config; scrape_url for one-shot extraction.
|
|
159
|
-
Follow envelope next_step and guidance. Botasaurus needs BOTASAURUS_SCRAPER_URL in this process env (boolean at html2rss://runtime; the URL is never returned).
|
|
160
|
-
TEXT
|
|
161
|
-
end
|
|
162
|
-
|
|
163
|
-
def botasaurus_configured?
|
|
164
|
-
!ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
|
|
205
|
+
def instructions_text
|
|
206
|
+
Outcome::Playbook.instructions
|
|
165
207
|
end
|
|
166
208
|
|
|
167
209
|
def tool_error_response(error)
|
|
@@ -175,7 +217,7 @@ module Html2rss
|
|
|
175
217
|
tool_error_response(error)
|
|
176
218
|
end
|
|
177
219
|
|
|
178
|
-
# rubocop:disable Metrics/MethodLength -- listing fields stay together
|
|
220
|
+
# rubocop:disable-next Metrics/MethodLength -- listing fields stay together
|
|
179
221
|
def define_envelope_tool(server, name:, description:, input_schema:,
|
|
180
222
|
annotations: Contract::ANNOTATIONS_OPEN_WORLD)
|
|
181
223
|
run = method(:handle_tool_call)
|
|
@@ -190,201 +232,38 @@ module Html2rss
|
|
|
190
232
|
run.call { yield(**kwargs) }
|
|
191
233
|
end
|
|
192
234
|
end
|
|
193
|
-
# rubocop:enable Metrics/MethodLength
|
|
194
235
|
|
|
195
236
|
def register_tools(server)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
'strategy "auto" triggers fallback chain (faraday → botasaurus) for JS-rendered sites.',
|
|
210
|
-
input_schema: Contract::SCRAPE_INPUT_SCHEMA
|
|
211
|
-
) do |server_context:, url:, strategy: 'auto', limit: 25, items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
212
|
-
scrape_outcome(url:, strategy:, limit:, items_selector:)
|
|
213
|
-
end
|
|
214
|
-
end
|
|
215
|
-
|
|
216
|
-
def scrape_outcome(url:, strategy:, limit:, items_selector:)
|
|
217
|
-
plan = (strategy || :auto).to_sym
|
|
218
|
-
feed_result = Html2rss.auto_feed_result(url, strategy: plan, limit:, items_selector:)
|
|
219
|
-
feed = feed_result.to_json_feed
|
|
220
|
-
Outcome.scrape(
|
|
221
|
-
items: feed[:items] || [],
|
|
222
|
-
requested_strategy: plan,
|
|
223
|
-
channel_title: feed[:title],
|
|
224
|
-
admission_drops: feed_result.status.admission_drops,
|
|
225
|
-
botasaurus_configured: botasaurus_configured?
|
|
226
|
-
)
|
|
227
|
-
end
|
|
228
|
-
|
|
229
|
-
def register_inspect_url(server)
|
|
230
|
-
define_envelope_tool(
|
|
231
|
-
server,
|
|
232
|
-
name: 'inspect_url',
|
|
233
|
-
description: 'Diagnostic page analysis (scrapers, SST, segments) plus recon: ' \
|
|
234
|
-
'final_url, status, scheme_downgrade, rel=alternate RSS/Atom feeds. ' \
|
|
235
|
-
'Use when scrape/capture is weak or you need those recon facts.',
|
|
236
|
-
input_schema: Contract::INSPECT_INPUT_SCHEMA
|
|
237
|
-
) do |server_context:, url:, strategy: 'auto'| # rubocop:disable Lint/UnusedBlockArgument
|
|
238
|
-
Outcome.inspect(payload: Inspect.call(url:, strategy:))
|
|
239
|
-
end
|
|
240
|
-
end
|
|
241
|
-
|
|
242
|
-
def register_capture_config(server) # rubocop:disable Metrics/MethodLength
|
|
243
|
-
define_envelope_tool(
|
|
244
|
-
server,
|
|
245
|
-
name: 'capture_config',
|
|
246
|
-
description: 'Derive a reusable html2rss feed config from a URL. ' \
|
|
247
|
-
'Use when the goal is a durable YAML (then validate_config). ' \
|
|
248
|
-
'Returns YAML inside payload.yaml (same serializer as CLI capture). ' \
|
|
249
|
-
'Draft only — catalog feeds still need directory.topics and title/url; ' \
|
|
250
|
-
'strive enhance: true. Full schema options live in resource html2rss://schema.',
|
|
251
|
-
input_schema: Contract::CAPTURE_INPUT_SCHEMA
|
|
252
|
-
) do |server_context:, url:, strategy: 'auto', items_selector: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
253
|
-
capture_outcome(url:, strategy:, items_selector:)
|
|
254
|
-
end
|
|
255
|
-
end
|
|
256
|
-
|
|
257
|
-
def capture_outcome(url:, strategy:, items_selector:) # rubocop:disable Metrics/MethodLength -- CaptureResult maps 1:1 onto Outcome
|
|
258
|
-
plan = (strategy || :auto).to_sym
|
|
259
|
-
result = Html2rss::Capture.build(url, strategy: plan, items_selector:)
|
|
260
|
-
Outcome.capture(
|
|
261
|
-
yaml: Config.to_yaml(result.config),
|
|
262
|
-
articles_count: result.articles_count,
|
|
263
|
-
has_selectors: result.has_selectors,
|
|
264
|
-
channel_title: result.channel_title,
|
|
265
|
-
requested_strategy: plan,
|
|
266
|
-
segment_strategy: result.segment_strategy,
|
|
267
|
-
selected_strategy: result.selected_strategy,
|
|
268
|
-
admission_drops: result.admission_drops
|
|
269
|
-
)
|
|
270
|
-
end
|
|
271
|
-
|
|
272
|
-
def register_validate_config(server) # rubocop:disable Metrics/MethodLength -- description is the published contract
|
|
273
|
-
define_envelope_tool(
|
|
274
|
-
server,
|
|
275
|
-
name: 'validate_config',
|
|
276
|
-
description: 'Validate a feed config hash XOR yaml string against the html2rss JSON schema. ' \
|
|
277
|
-
'Call before apply_config. Failures return isError with payload.errors. ' \
|
|
278
|
-
'Full schema lives in resource html2rss://schema.',
|
|
279
|
-
input_schema: Contract::CONFIG_XOR_SCHEMA,
|
|
280
|
-
annotations: Contract::ANNOTATIONS_VALIDATE
|
|
281
|
-
) do |server_context:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
282
|
-
validate_outcome(config:, yaml:)
|
|
283
|
-
end
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
def validate_outcome(config:, yaml:)
|
|
287
|
-
validation = Html2rss::Config.validate(ConfigArgument.parse(config:, yaml:).config)
|
|
288
|
-
Outcome.validate(errors: validation.success? ? nil : validation.errors.to_h)
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
def register_apply_config(server)
|
|
292
|
-
define_envelope_tool(
|
|
293
|
-
server,
|
|
294
|
-
name: 'apply_config',
|
|
295
|
-
description: 'Apply a validated feed config (hash XOR yaml) and return RSS XML in payload.rss. ' \
|
|
296
|
-
'isError when the feed has zero items (ship gate). payload.item_count is RSS item count. ' \
|
|
297
|
-
'Use after validate_config succeeds.',
|
|
298
|
-
input_schema: Contract::APPLY_INPUT_SCHEMA
|
|
299
|
-
) do |server_context:, url:, config: nil, yaml: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
300
|
-
apply_outcome(url:, config:, yaml:)
|
|
237
|
+
Tools.register_all(server, registrar: method(:define_envelope_tool))
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def register_resources(server)
|
|
241
|
+
RESOURCES.each do |entry|
|
|
242
|
+
server.define_resource(
|
|
243
|
+
uri: entry[:uri],
|
|
244
|
+
name: entry[:name],
|
|
245
|
+
description: entry[:description],
|
|
246
|
+
mime_type: entry[:mime_type]
|
|
247
|
+
) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
248
|
+
entry[:body].call
|
|
249
|
+
end
|
|
301
250
|
end
|
|
302
251
|
end
|
|
303
252
|
|
|
304
|
-
def
|
|
305
|
-
feed_config = ConfigArgument.parse(config:, yaml:).config
|
|
306
|
-
feed_config[:channel] ||= {}
|
|
307
|
-
feed_config[:channel][:url] ||= url
|
|
308
|
-
feed_result = Html2rss.feed_result(feed_config)
|
|
309
|
-
rss = feed_result.to_rss
|
|
310
|
-
Outcome.apply(rss: rss.to_s, item_count: rss.items.size, empty: feed_result.empty?)
|
|
311
|
-
end
|
|
312
|
-
|
|
313
|
-
def register_resources(server) # rubocop:disable Metrics/MethodLength
|
|
314
|
-
configured = method(:botasaurus_configured?)
|
|
315
|
-
server.define_resource(
|
|
316
|
-
uri: 'html2rss://schema',
|
|
317
|
-
name: 'Configuration JSON Schema',
|
|
318
|
-
description: 'Full JSON Schema for html2rss feed configurations',
|
|
319
|
-
mime_type: 'application/json'
|
|
320
|
-
) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
321
|
-
schema = Html2rss::Config.json_schema_json(pretty: true)
|
|
322
|
-
[{ uri: 'html2rss://schema', mimeType: 'application/json', text: schema }]
|
|
323
|
-
end
|
|
324
|
-
|
|
325
|
-
server.define_resource(
|
|
326
|
-
uri: 'html2rss://extractors',
|
|
327
|
-
name: 'Available Extractors',
|
|
328
|
-
description: 'Registered extractor names for selector configs ' \
|
|
329
|
-
'(full option docs live in html2rss://schema $defs)',
|
|
330
|
-
mime_type: 'application/json'
|
|
331
|
-
) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
332
|
-
extractors = Html2rss::Selectors::Extractors::NAME_TO_CLASS.keys.map(&:to_s).sort
|
|
333
|
-
[{ uri: 'html2rss://extractors', mimeType: 'application/json',
|
|
334
|
-
text: JSON.pretty_generate(extractors) }]
|
|
335
|
-
end
|
|
336
|
-
|
|
337
|
-
server.define_resource(
|
|
338
|
-
uri: 'html2rss://strategies',
|
|
339
|
-
name: 'Available Strategies',
|
|
340
|
-
description: 'Published MCP request strategy names',
|
|
341
|
-
mime_type: 'application/json'
|
|
342
|
-
) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
343
|
-
[{ uri: 'html2rss://strategies', mimeType: 'application/json',
|
|
344
|
-
text: JSON.generate(Contract::STRATEGIES) }]
|
|
345
|
-
end
|
|
346
|
-
|
|
347
|
-
server.define_resource(
|
|
348
|
-
uri: 'html2rss://runtime',
|
|
349
|
-
name: 'Runtime capabilities',
|
|
350
|
-
description: 'Whether optional transports are configured in this MCP process (never leaks secrets)',
|
|
351
|
-
mime_type: 'application/json'
|
|
352
|
-
) do |server_context: nil| # rubocop:disable Lint/UnusedBlockArgument
|
|
353
|
-
[{ uri: 'html2rss://runtime', mimeType: 'application/json',
|
|
354
|
-
text: JSON.pretty_generate(botasaurus_configured: configured.call) }]
|
|
355
|
-
end
|
|
356
|
-
end
|
|
357
|
-
|
|
358
|
-
def register_prompts(server)
|
|
359
|
-
register_scrape_webpage_prompt(server)
|
|
360
|
-
register_capture_feed_config_prompt(server)
|
|
361
|
-
end
|
|
362
|
-
|
|
363
|
-
def register_scrape_webpage_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
|
|
253
|
+
def register_prompts(server) # rubocop:disable Metrics/MethodLength -- prompt argument mapping
|
|
364
254
|
to_result = method(:prompt_result)
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
def register_capture_feed_config_prompt(server) # rubocop:disable Metrics/MethodLength -- SDK prompt types stay together
|
|
378
|
-
to_result = method(:prompt_result)
|
|
379
|
-
text_for = method(:capture_feed_config_text)
|
|
380
|
-
server.define_prompt(
|
|
381
|
-
name: 'capture-feed-config',
|
|
382
|
-
description: 'Guided capture → validate → apply; YAML draft plus catalog rewrite',
|
|
383
|
-
arguments: [
|
|
384
|
-
::MCP::Prompt::Argument.new(name: 'url', description: 'URL to analyze', required: true)
|
|
385
|
-
]
|
|
386
|
-
) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
|
|
387
|
-
to_result.call(text_for.call(args.fetch(:url)))
|
|
255
|
+
PROMPTS.each do |entry|
|
|
256
|
+
arguments = entry[:arguments].map do |spec|
|
|
257
|
+
::MCP::Prompt::Argument.new(name: spec[:name], description: spec[:description],
|
|
258
|
+
required: spec.fetch(:required, false))
|
|
259
|
+
end
|
|
260
|
+
server.define_prompt(
|
|
261
|
+
name: entry[:name],
|
|
262
|
+
description: entry[:description],
|
|
263
|
+
arguments:
|
|
264
|
+
) do |args, server_context:| # rubocop:disable Lint/UnusedBlockArgument
|
|
265
|
+
to_result.call(entry[:body].call(args))
|
|
266
|
+
end
|
|
388
267
|
end
|
|
389
268
|
end
|
|
390
269
|
|
|
@@ -395,26 +274,6 @@ module Html2rss
|
|
|
395
274
|
]
|
|
396
275
|
)
|
|
397
276
|
end
|
|
398
|
-
|
|
399
|
-
def scrape_webpage_text(url)
|
|
400
|
-
<<~MSG.strip
|
|
401
|
-
Scrape #{url} with scrape_url (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
|
|
402
|
-
Follow envelope next_step and guidance. Call inspect_url only if articles are empty/weak or you need recon (final_url, status, scheme_downgrade, alternate_feeds).
|
|
403
|
-
Do not retry scrape_url with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
|
|
404
|
-
Return payload.items (not a raw JSON array).
|
|
405
|
-
MSG
|
|
406
|
-
end
|
|
407
|
-
|
|
408
|
-
def capture_feed_config_text(url)
|
|
409
|
-
<<~MSG.strip
|
|
410
|
-
Build a reusable html2rss feed config for #{url}:
|
|
411
|
-
1) capture_config — YAML is payload.yaml. Check payload.articles_count and payload.has_selectors. Strive to keep enhance: true (false only when chrome leaks into items).
|
|
412
|
-
2) Follow next_step. If weak or you need recon, inspect_url. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
|
|
413
|
-
3) validate_config with yaml (or config hash) — must not be isError
|
|
414
|
-
4) apply_config — isError if zero items. Confirm payload.item_count before shipping.
|
|
415
|
-
If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
|
|
416
|
-
MSG
|
|
417
|
-
end
|
|
418
277
|
end
|
|
419
278
|
end
|
|
420
279
|
end
|