html2rss 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +82 -25
- data/html2rss.gemspec +0 -1
- data/lib/html2rss/article.rb +1 -2
- data/lib/html2rss/auto_source/README.md +8 -8
- data/lib/html2rss/auto_source/cleanup.rb +92 -1
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
- data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
- data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
- data/lib/html2rss/auto_source/scraper.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
- data/lib/html2rss/auto_source/segmenter.rb +1 -2
- data/lib/html2rss/auto_source.rb +25 -11
- data/lib/html2rss/batch.rb +169 -0
- data/lib/html2rss/capture/README.md +15 -6
- data/lib/html2rss/capture.rb +158 -15
- data/lib/html2rss/channel.rb +1 -2
- data/lib/html2rss/cli/probe_view.rb +66 -0
- data/lib/html2rss/cli/render.rb +124 -0
- data/lib/html2rss/cli/validate.rb +91 -0
- data/lib/html2rss/cli.rb +318 -193
- data/lib/html2rss/config/request_headers.rb +4 -14
- data/lib/html2rss/config/schema.rb +7 -5
- data/lib/html2rss/config/validator.rb +1 -0
- data/lib/html2rss/config.rb +59 -22
- data/lib/html2rss/doctor/botasaurus.rb +130 -0
- data/lib/html2rss/feed_builder/rss.rb +1 -2
- data/lib/html2rss/feed_pipeline/README.md +5 -5
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
- data/lib/html2rss/feed_pipeline.rb +27 -16
- data/lib/html2rss/feed_resolution/README.md +11 -11
- data/lib/html2rss/feed_resolution.rb +6 -12
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
- data/lib/html2rss/html/article_extractor.rb +4 -9
- data/lib/html2rss/html/feed_link.rb +4 -2
- data/lib/html2rss/html/navigator.rb +4 -39
- data/lib/html2rss/html/probe.rb +82 -0
- data/lib/html2rss/html/sst_article_extractor.rb +1 -2
- data/lib/html2rss/link_destination/noise_policy.rb +1 -2
- data/lib/html2rss/link_destination/path_classifier.rb +20 -19
- data/lib/html2rss/mcp/README.md +80 -0
- data/lib/html2rss/mcp/contract.rb +135 -13
- data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
- data/lib/html2rss/mcp/outcome.rb +112 -42
- data/lib/html2rss/mcp/runtime.rb +45 -0
- data/lib/html2rss/mcp/server/tools.rb +277 -0
- data/lib/html2rss/mcp/server.rb +91 -232
- data/lib/html2rss/mcp.rb +3 -0
- data/lib/html2rss/page_recon/README.md +69 -0
- data/lib/html2rss/page_recon/diagnostics.rb +211 -0
- data/lib/html2rss/page_recon.rb +56 -2
- data/lib/html2rss/recon.rb +252 -0
- data/lib/html2rss/request_service/blocked_surface.rb +1 -0
- data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
- data/lib/html2rss/request_service/policy.rb +1 -2
- data/lib/html2rss/request_service/response.rb +3 -4
- data/lib/html2rss/request_service/strategy.rb +1 -2
- data/lib/html2rss/request_service.rb +2 -0
- data/lib/html2rss/request_session.rb +1 -2
- data/lib/html2rss/scrape_target.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
- data/lib/html2rss/selectors/post_processors.rb +0 -1
- data/lib/html2rss/selectors.rb +33 -10
- data/lib/html2rss/sst/attrs.rb +1 -2
- data/lib/html2rss/sst/normalizer.rb +49 -11
- data/lib/html2rss/status.rb +3 -6
- data/lib/html2rss/syndication/README.md +6 -6
- data/lib/html2rss/syndication/discovery.rb +2 -4
- data/lib/html2rss/test/enhance_audit.rb +192 -0
- data/lib/html2rss/test.rb +458 -0
- data/lib/html2rss/url.rb +1 -2
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +157 -35
- data/schema/html2rss-config.schema.json +9 -29
- metadata +17 -18
- data/lib/html2rss/mcp/inspect.rb +0 -138
- data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
3
5
|
module Html2rss
|
|
4
6
|
module MCP
|
|
5
7
|
##
|
|
@@ -9,6 +11,10 @@ module Html2rss
|
|
|
9
11
|
# Published MCP request strategies (excludes +local_file+).
|
|
10
12
|
STRATEGIES = %w[auto faraday botasaurus].freeze
|
|
11
13
|
|
|
14
|
+
# Bump when tool names, required inputs, or envelope semantics change (independent of gem +VERSION+).
|
|
15
|
+
MCP_CONTRACT_VERSION = 2
|
|
16
|
+
public_constant :MCP_CONTRACT_VERSION
|
|
17
|
+
|
|
12
18
|
# Raised when apply/validate config uses an unpublished MCP request adapter.
|
|
13
19
|
class UnpublishedRequestError < ArgumentError; end
|
|
14
20
|
|
|
@@ -51,14 +57,14 @@ module Html2rss
|
|
|
51
57
|
}.freeze
|
|
52
58
|
}.freeze
|
|
53
59
|
|
|
54
|
-
# Input schema for +
|
|
60
|
+
# Input schema for +validate+ (config XOR yaml).
|
|
55
61
|
CONFIG_XOR_SCHEMA = {
|
|
56
62
|
type: 'object',
|
|
57
63
|
properties: CONFIG_XOR_PROPERTIES,
|
|
58
64
|
oneOf: XOR_ONE_OF
|
|
59
65
|
}.freeze
|
|
60
66
|
|
|
61
|
-
# Input schema for +
|
|
67
|
+
# Input schema for +apply+ (required URL plus config XOR yaml).
|
|
62
68
|
APPLY_INPUT_SCHEMA = {
|
|
63
69
|
type: 'object',
|
|
64
70
|
properties: { url: URL_PROPERTY, **CONFIG_XOR_PROPERTIES }.freeze,
|
|
@@ -66,7 +72,28 @@ module Html2rss
|
|
|
66
72
|
oneOf: XOR_ONE_OF
|
|
67
73
|
}.freeze
|
|
68
74
|
|
|
69
|
-
# Input schema for +
|
|
75
|
+
# Input schema for +test+.
|
|
76
|
+
TEST_INPUT_SCHEMA = {
|
|
77
|
+
type: 'object',
|
|
78
|
+
properties: {
|
|
79
|
+
**CONFIG_XOR_PROPERTIES,
|
|
80
|
+
min_items: { type: 'integer', description: 'Minimum required items (default: 1)', default: 1 },
|
|
81
|
+
strict_quality: {
|
|
82
|
+
type: 'boolean',
|
|
83
|
+
description: 'Fail when ship-quality audit thresholds are exceeded (default: false)',
|
|
84
|
+
default: false
|
|
85
|
+
},
|
|
86
|
+
compare_enhance: {
|
|
87
|
+
type: 'boolean',
|
|
88
|
+
description: 'Diagnostic: compare extraction with enhance on vs off on cached HTML (default: false)',
|
|
89
|
+
default: false
|
|
90
|
+
},
|
|
91
|
+
strategy: STRATEGY_PROPERTY
|
|
92
|
+
}.freeze,
|
|
93
|
+
oneOf: XOR_ONE_OF
|
|
94
|
+
}.freeze
|
|
95
|
+
|
|
96
|
+
# Input schema for +scrape+.
|
|
70
97
|
SCRAPE_INPUT_SCHEMA = {
|
|
71
98
|
type: 'object',
|
|
72
99
|
properties: {
|
|
@@ -78,24 +105,87 @@ module Html2rss
|
|
|
78
105
|
required: %w[url]
|
|
79
106
|
}.freeze
|
|
80
107
|
|
|
81
|
-
# Input schema for +
|
|
108
|
+
# Input schema for +inspect+.
|
|
82
109
|
INSPECT_INPUT_SCHEMA = {
|
|
83
110
|
type: 'object',
|
|
84
111
|
properties: { url: URL_PROPERTY, strategy: INSPECT_STRATEGY_PROPERTY }.freeze,
|
|
85
112
|
required: %w[url]
|
|
86
113
|
}.freeze
|
|
87
114
|
|
|
88
|
-
# Input schema for +
|
|
115
|
+
# Input schema for +recon+.
|
|
116
|
+
RECON_INPUT_SCHEMA = INSPECT_INPUT_SCHEMA
|
|
117
|
+
|
|
118
|
+
# Input schema for +capture+.
|
|
89
119
|
CAPTURE_INPUT_SCHEMA = {
|
|
90
120
|
type: 'object',
|
|
91
121
|
properties: {
|
|
92
122
|
url: URL_PROPERTY,
|
|
93
123
|
strategy: STRATEGY_PROPERTY,
|
|
94
|
-
items_selector: { type: 'string', description: 'Optional CSS selector hint for items' }
|
|
124
|
+
items_selector: { type: 'string', description: 'Optional CSS selector hint for items' },
|
|
125
|
+
force: { type: 'boolean', description: 'Bypass native feed check', default: false },
|
|
126
|
+
topics: {
|
|
127
|
+
type: 'array',
|
|
128
|
+
items: { type: 'string' },
|
|
129
|
+
description: 'Directory topics override'
|
|
130
|
+
}.freeze,
|
|
131
|
+
title: { type: 'string', description: 'Channel title override' },
|
|
132
|
+
summary: { type: 'string', description: 'Directory summary override' },
|
|
133
|
+
enhance: { type: 'boolean', description: 'Force enhance on or off' },
|
|
134
|
+
limit: { type: 'integer', description: 'Max articles to keep' },
|
|
135
|
+
max_redirects: { type: 'integer', description: 'Optional redirect limit override' },
|
|
136
|
+
max_requests: { type: 'integer', description: 'Optional request budget override' }
|
|
95
137
|
}.freeze,
|
|
96
138
|
required: %w[url]
|
|
97
139
|
}.freeze
|
|
98
140
|
|
|
141
|
+
##
|
|
142
|
+
# Shared JSON Schema for batch URL tools (+batch_scrape+, +batch_inspect+, +batch_recon+).
|
|
143
|
+
#
|
|
144
|
+
# @param urls_description [String] description for the +urls+ array property
|
|
145
|
+
# @param strategy_property [Hash] strategy JSON Schema property
|
|
146
|
+
# @param extra_properties [Hash] additional tool-specific properties (e.g. +limit+ on scrape)
|
|
147
|
+
# @return [Hash]
|
|
148
|
+
def self.batch_urls_input_schema(urls_description:, strategy_property:, extra_properties: {}) # rubocop:disable Metrics/MethodLength
|
|
149
|
+
{
|
|
150
|
+
type: 'object',
|
|
151
|
+
properties: {
|
|
152
|
+
urls: {
|
|
153
|
+
type: 'array',
|
|
154
|
+
items: URL_PROPERTY,
|
|
155
|
+
minItems: 1,
|
|
156
|
+
maxItems: 25,
|
|
157
|
+
description: urls_description
|
|
158
|
+
}.freeze,
|
|
159
|
+
strategy: strategy_property,
|
|
160
|
+
concurrency: {
|
|
161
|
+
type: 'integer',
|
|
162
|
+
description: 'Max parallel worker threads (1..10, default: 5)',
|
|
163
|
+
default: 5
|
|
164
|
+
},
|
|
165
|
+
**extra_properties
|
|
166
|
+
}.freeze,
|
|
167
|
+
required: %w[urls]
|
|
168
|
+
}
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Input schema for +batch_scrape+.
|
|
172
|
+
BATCH_SCRAPE_INPUT_SCHEMA = batch_urls_input_schema(
|
|
173
|
+
urls_description: 'List of page URLs to scrape (1..25)',
|
|
174
|
+
strategy_property: STRATEGY_PROPERTY,
|
|
175
|
+
extra_properties: {
|
|
176
|
+
limit: { type: 'integer', description: 'Max articles per URL to keep (default 10)', default: 10 }
|
|
177
|
+
}
|
|
178
|
+
).freeze
|
|
179
|
+
|
|
180
|
+
# Input schema for +batch_inspect+.
|
|
181
|
+
BATCH_INSPECT_INPUT_SCHEMA = batch_urls_input_schema(
|
|
182
|
+
urls_description: 'List of page URLs to inspect (1..25)',
|
|
183
|
+
strategy_property: INSPECT_STRATEGY_PROPERTY
|
|
184
|
+
).freeze
|
|
185
|
+
|
|
186
|
+
# Input schema for +batch_recon+.
|
|
187
|
+
BATCH_RECON_INPUT_SCHEMA = BATCH_INSPECT_INPUT_SCHEMA
|
|
188
|
+
|
|
99
189
|
# Tool annotations for open-world read-only tools.
|
|
100
190
|
ANNOTATIONS_OPEN_WORLD = {
|
|
101
191
|
read_only_hint: true,
|
|
@@ -104,19 +194,51 @@ module Html2rss
|
|
|
104
194
|
open_world_hint: true
|
|
105
195
|
}.freeze
|
|
106
196
|
|
|
107
|
-
# Tool annotations for +
|
|
197
|
+
# Tool annotations for +validate+ (closed world).
|
|
108
198
|
ANNOTATIONS_VALIDATE = ANNOTATIONS_OPEN_WORLD.merge(open_world_hint: false).freeze
|
|
109
199
|
|
|
110
200
|
# Human titles for +tools/list+.
|
|
111
201
|
TITLES = {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
202
|
+
scrape: 'Scrape',
|
|
203
|
+
inspect: 'Inspect',
|
|
204
|
+
recon: 'Recon',
|
|
205
|
+
batch_scrape: 'Batch scrape',
|
|
206
|
+
batch_inspect: 'Batch inspect',
|
|
207
|
+
batch_recon: 'Batch recon',
|
|
208
|
+
capture: 'Capture',
|
|
209
|
+
validate: 'Validate',
|
|
210
|
+
apply: 'Apply',
|
|
211
|
+
test: 'Test'
|
|
117
212
|
}.freeze
|
|
118
213
|
|
|
214
|
+
# @api private
|
|
215
|
+
CATALOG_ENTRY_LINE = lambda do |entry|
|
|
216
|
+
schema = entry.fetch(:input_schema)
|
|
217
|
+
required = Array(schema[:required]).sort.join(',')
|
|
218
|
+
one_of = Array(schema[:oneOf]).map { |branch| Array(branch[:required]).sort.join('+') }.sort.join('|')
|
|
219
|
+
[entry.fetch(:name), required, one_of].reject(&:empty?).join(':')
|
|
220
|
+
end.freeze
|
|
221
|
+
|
|
119
222
|
class << self
|
|
223
|
+
##
|
|
224
|
+
# Canonical MCP tool names in alphabetical order (same set as +tools/list+).
|
|
225
|
+
#
|
|
226
|
+
# @return [Array<String>]
|
|
227
|
+
def catalog_tools
|
|
228
|
+
Server::Tools::TOOLS.map { |entry| entry.fetch(:name) }.sort
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
##
|
|
232
|
+
# Stable fingerprint of published tools, required keys, and +oneOf+ branches.
|
|
233
|
+
# Clients compare against a cached +tools/list+ to detect stale catalogs.
|
|
234
|
+
# Bump {MCP_CONTRACT_VERSION} for envelope or breaking wire semantics only.
|
|
235
|
+
#
|
|
236
|
+
# @return [String] 16-char hex digest prefix
|
|
237
|
+
def catalog_fingerprint
|
|
238
|
+
lines = Server::Tools::TOOLS.sort_by { |entry| entry.fetch(:name) }.map(&CATALOG_ENTRY_LINE)
|
|
239
|
+
Digest::SHA256.hexdigest(lines.join("\n")).slice(0, 16)
|
|
240
|
+
end
|
|
241
|
+
|
|
120
242
|
##
|
|
121
243
|
# Envelope JSON Schema. Built lazily so Zeitwerk can load Contract before Outcome.
|
|
122
244
|
#
|
|
@@ -151,7 +273,7 @@ module Html2rss
|
|
|
151
273
|
|
|
152
274
|
##
|
|
153
275
|
# Rejects unpublished MCP request adapters so apply/validate cannot
|
|
154
|
-
#
|
|
276
|
+
# +File.read+ arbitrary paths. CLI and Config still allow +local_file+.
|
|
155
277
|
#
|
|
156
278
|
# @param config [Hash]
|
|
157
279
|
# @return [void]
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
module MCP
|
|
5
|
+
class Outcome
|
|
6
|
+
##
|
|
7
|
+
# Single source of truth for MCP agent instructions, next-step guidance,
|
|
8
|
+
# and prompt bodies. {Server} delegates here — do not duplicate prose in
|
|
9
|
+
# +server.rb+.
|
|
10
|
+
class Playbook
|
|
11
|
+
# Default guidance copy keyed by {Outcome::NextStep::NAMES}.
|
|
12
|
+
GUIDANCE = {
|
|
13
|
+
done: 'Done. Read payload for the result.',
|
|
14
|
+
inspect: 'Call inspect next. Read payload for diagnostics (final_url, status, ' \
|
|
15
|
+
'scheme_downgrade, alternate_feeds, likely_js_shell, redirect_summary).',
|
|
16
|
+
recon: 'Call recon next. Read payload for verdict and native_feed preference.',
|
|
17
|
+
validate: 'Call validate with payload.yaml or a config hash (XOR, not both).',
|
|
18
|
+
apply: 'Call apply next. Confirm payload.item_count before shipping.',
|
|
19
|
+
scrape: 'Call scrape for articles now. strategy auto already runs Faraday then Botasaurus ' \
|
|
20
|
+
'and promotes native RSS/Atom when present.',
|
|
21
|
+
capture: 'Call capture for a reusable YAML draft, then follow next_step.',
|
|
22
|
+
read_runtime: 'Read html2rss://runtime. Compare mcp_contract_version and catalog_fingerprint ' \
|
|
23
|
+
'to your cached tools/list before retrying unknown tools. ' \
|
|
24
|
+
'Set BOTASAURUS_SCRAPER_URL on the MCP process if botasaurus_configured is false.',
|
|
25
|
+
test: 'Call test next (schema + live extraction). Confirm payload.item_count, ' \
|
|
26
|
+
'failure_kind, and payload.quality_report warnings before shipping.'
|
|
27
|
+
}.freeze
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
##
|
|
31
|
+
# Published MCP server instructions (decision tree for agents).
|
|
32
|
+
#
|
|
33
|
+
# @return [String]
|
|
34
|
+
def instructions # rubocop:disable Metrics/MethodLength -- agent decision tree is the published contract
|
|
35
|
+
<<~TEXT.strip
|
|
36
|
+
html2rss MCP — decide which tool to call:
|
|
37
|
+
|
|
38
|
+
1. Need articles now (no saved config)? → scrape (or batch_scrape for multiple)
|
|
39
|
+
- strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
|
|
40
|
+
- Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
|
|
41
|
+
2. Need a reusable feed YAML? → capture → test → apply
|
|
42
|
+
- capture returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Default enhance follows capture evidence (false when admission_drops show chrome); override only when needed.
|
|
43
|
+
- test runs schema + live extraction (min items). apply is the ship gate (isError on zero items). Confirm payload.item_count and payload.quality_report warnings (including enhance_gains when selectors.items.enhance is true). Use compare_enhance on test for enhance on/off diagnostics.
|
|
44
|
+
- validate alone is for schema-only checks; on success next_step is test.
|
|
45
|
+
3. Weak scrape/capture or recon (final URL, status, https→http, rel=alternate feeds)? → inspect (or batch_inspect). Read likely_js_shell vs blocked_surface when articles_count is 0. When alternates warrant it, inspect next_step is recon.
|
|
46
|
+
4. Have a config already? → validate (must succeed) → test → apply
|
|
47
|
+
5. Schema / extractors / strategies / runtime → resources html2rss://schema|extractors|strategies|runtime
|
|
48
|
+
- runtime publishes version, mcp_contract_version, catalog_fingerprint, tools, botasaurus_configured.
|
|
49
|
+
- Refresh tools/list when catalog_fingerprint differs from your cache.
|
|
50
|
+
|
|
51
|
+
Prefer capture for durable config; scrape / batch_scrape for one-shot extraction.
|
|
52
|
+
Follow envelope next_step and guidance. Botasaurus needs BOTASAURUS_SCRAPER_URL in this process env (read html2rss://runtime; the URL is never returned).
|
|
53
|
+
TEXT
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
##
|
|
57
|
+
# @param url [String]
|
|
58
|
+
# @return [String]
|
|
59
|
+
def scrape_webpage_prompt(url)
|
|
60
|
+
<<~MSG.strip
|
|
61
|
+
Scrape #{url} with scrape (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
|
|
62
|
+
Follow envelope next_step and guidance. Call inspect only if articles are empty/weak or you need diagnostics (final_url, status, scheme_downgrade, alternate_feeds). When inspect finds alternates, follow next_step to recon.
|
|
63
|
+
Do not retry scrape with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
|
|
64
|
+
Return payload.items (not a raw JSON array).
|
|
65
|
+
MSG
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
##
|
|
69
|
+
# @param url [String]
|
|
70
|
+
# @return [String]
|
|
71
|
+
def capture_feed_config_prompt(url)
|
|
72
|
+
<<~MSG.strip
|
|
73
|
+
Build a reusable html2rss feed config for #{url}:
|
|
74
|
+
1) capture — YAML is payload.yaml. Check payload.articles_count, payload.has_selectors, and payload.suggested_channel_url. enhance defaults from admission evidence (false when chrome drops are high). When payload.native_feed is set, follow next_step (done — use the native feed).
|
|
75
|
+
2) Follow next_step. If weak or you need recon, inspect then recon when alternates warrant it. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
|
|
76
|
+
3) test with yaml (or config hash) — schema + live extraction. On :schema failure, validate; on :execution/:min_items, recapture. Read payload.quality_report.enhance_gains when enhance is on; optional compare_enhance compares enhance off vs on without changing shipped RSS.
|
|
77
|
+
4) apply — isError if zero items. Confirm payload.item_count and payload.quality_report (including enhance_gains) before shipping.
|
|
78
|
+
If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
|
|
79
|
+
MSG
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
##
|
|
83
|
+
# @param report [PageRecon::Diagnostics::Report]
|
|
84
|
+
# @return [String]
|
|
85
|
+
def inspect_guidance(report)
|
|
86
|
+
return GUIDANCE.fetch(:inspect) unless report.articles_count.zero?
|
|
87
|
+
|
|
88
|
+
empty_extract_guidance(report.data)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
##
|
|
92
|
+
# @param data [Hash{Symbol => Object}]
|
|
93
|
+
# @return [String]
|
|
94
|
+
def empty_extract_guidance(data) # rubocop:disable Metrics/MethodLength
|
|
95
|
+
if data[:blocked_surface] || data[:surface_category].to_s == 'blocked_surface'
|
|
96
|
+
return 'Blocked or anti-bot interstitial likely. Retry scrape with strategy botasaurus once ' \
|
|
97
|
+
'(or CLI inspect --deep when BOTASAURUS_SCRAPER_URL is set). ' \
|
|
98
|
+
'Do not retry explicit faraday after auto.'
|
|
99
|
+
end
|
|
100
|
+
if data[:likely_js_shell]
|
|
101
|
+
return 'JS-rendered shell likely (html_present, zero articles). Use strategy auto or botasaurus; ' \
|
|
102
|
+
'CLI inspect --deep for one Botasaurus diagnostic hop.'
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
'Empty extract on a static-looking page. Verify redirect_summary.final_url and surface; ' \
|
|
106
|
+
'capture may need selector hints.'
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
##
|
|
110
|
+
# @param result [Html2rss::Recon::Result]
|
|
111
|
+
# @param next_step [Outcome::NextStep]
|
|
112
|
+
# @return [String]
|
|
113
|
+
def recon_guidance(result, next_step)
|
|
114
|
+
return next_step.guidance unless result.scheme_downgrade
|
|
115
|
+
|
|
116
|
+
"#{next_step.guidance} HTTPS→HTTP downgrade detected: try one Botasaurus scrape before DROP."
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
data/lib/html2rss/mcp/outcome.rb
CHANGED
|
@@ -5,10 +5,10 @@ module Html2rss
|
|
|
5
5
|
Outcome = Data.define(:ok, :next_step, :guidance, :payload)
|
|
6
6
|
|
|
7
7
|
##
|
|
8
|
-
# Typed MCP tool result. Owns next-step policy
|
|
9
|
-
#
|
|
10
|
-
class Outcome
|
|
11
|
-
# Matches {ConfigArgument} XOR
|
|
8
|
+
# Typed MCP tool result. Owns next-step policy; guidance copy lives in
|
|
9
|
+
# {Playbook}.
|
|
10
|
+
class Outcome # rubocop:disable Metrics/ClassLength -- next-step policy + factories stay co-located
|
|
11
|
+
# Matches {ConfigArgument} XOR +ArgumentError+ messages.
|
|
12
12
|
XOR_ERROR = /exactly one of config or yaml/
|
|
13
13
|
NextStep = Data.define(:name, :guidance)
|
|
14
14
|
|
|
@@ -16,20 +16,7 @@ module Html2rss
|
|
|
16
16
|
# Closed set of agent next actions. Invalid names cannot be constructed.
|
|
17
17
|
class NextStep
|
|
18
18
|
# Wire names for +next_step+.
|
|
19
|
-
NAMES = %i[done
|
|
20
|
-
# Default guidance copy keyed by {NAMES}.
|
|
21
|
-
GUIDANCE = {
|
|
22
|
-
done: 'Done. Read payload for the result.',
|
|
23
|
-
inspect_url: 'Call inspect_url next. Read payload for recon (final_url, status, ' \
|
|
24
|
-
'scheme_downgrade, alternate_feeds).',
|
|
25
|
-
validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
|
|
26
|
-
apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
|
|
27
|
-
scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus ' \
|
|
28
|
-
'and promotes native RSS/Atom when present.',
|
|
29
|
-
capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
|
|
30
|
-
read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
|
|
31
|
-
'if botasaurus_configured is false.'
|
|
32
|
-
}.freeze
|
|
19
|
+
NAMES = %i[done inspect recon validate apply scrape capture read_runtime test].freeze
|
|
33
20
|
|
|
34
21
|
##
|
|
35
22
|
# @param name [Symbol, String]
|
|
@@ -38,7 +25,7 @@ module Html2rss
|
|
|
38
25
|
step = name.to_sym
|
|
39
26
|
raise ArgumentError, "unknown next_step: #{name.inspect}" unless NAMES.include?(step)
|
|
40
27
|
|
|
41
|
-
super(name: step, guidance: (guidance || GUIDANCE.fetch(step)).freeze)
|
|
28
|
+
super(name: step, guidance: (guidance || Playbook::GUIDANCE.fetch(step)).freeze)
|
|
42
29
|
end
|
|
43
30
|
|
|
44
31
|
class << self
|
|
@@ -79,11 +66,21 @@ module Html2rss
|
|
|
79
66
|
end
|
|
80
67
|
|
|
81
68
|
##
|
|
82
|
-
# @param
|
|
69
|
+
# @param report [PageRecon::Diagnostics::Report]
|
|
83
70
|
# @return [Outcome]
|
|
84
|
-
def inspect(
|
|
85
|
-
next_step = inspect_next_step(
|
|
86
|
-
|
|
71
|
+
def inspect(report:)
|
|
72
|
+
next_step = inspect_next_step(report)
|
|
73
|
+
guidance = Playbook.inspect_guidance(report)
|
|
74
|
+
new(ok: true, next_step:, guidance:, payload: report.to_wire_h)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
##
|
|
78
|
+
# @param result [Html2rss::Recon::Result]
|
|
79
|
+
# @return [Outcome]
|
|
80
|
+
def recon(result:)
|
|
81
|
+
next_step = recon_next_step(result)
|
|
82
|
+
guidance = Playbook.recon_guidance(result, next_step)
|
|
83
|
+
new(ok: true, next_step:, guidance:, payload: result.to_h)
|
|
87
84
|
end
|
|
88
85
|
|
|
89
86
|
##
|
|
@@ -95,13 +92,17 @@ module Html2rss
|
|
|
95
92
|
# @param segment_strategy [Symbol, String, nil]
|
|
96
93
|
# @param selected_strategy [Symbol, String, nil]
|
|
97
94
|
# @param admission_drops [Hash]
|
|
95
|
+
# @param native_feed [String, nil]
|
|
96
|
+
# @param suggested_channel_url [String, nil]
|
|
98
97
|
# @return [Outcome]
|
|
99
98
|
def capture(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
|
|
100
|
-
segment_strategy: nil, selected_strategy: nil, admission_drops: {}
|
|
101
|
-
|
|
99
|
+
segment_strategy: nil, selected_strategy: nil, admission_drops: {}, native_feed: nil,
|
|
100
|
+
suggested_channel_url: nil)
|
|
101
|
+
next_step = capture_next_step(articles_count:, has_selectors:, native_feed:)
|
|
102
102
|
new(ok: true, next_step:, guidance: next_step.guidance, payload: capture_payload(
|
|
103
103
|
yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:,
|
|
104
|
-
segment_strategy:, selected_strategy:, admission_drops
|
|
104
|
+
segment_strategy:, selected_strategy:, admission_drops:, native_feed:,
|
|
105
|
+
suggested_channel_url:
|
|
105
106
|
))
|
|
106
107
|
end
|
|
107
108
|
|
|
@@ -110,19 +111,50 @@ module Html2rss
|
|
|
110
111
|
# @return [Outcome]
|
|
111
112
|
def validate(errors:)
|
|
112
113
|
ok = errors.nil?
|
|
113
|
-
next_step = ok ? NextStep.
|
|
114
|
+
next_step = ok ? NextStep.test : NextStep.validate
|
|
114
115
|
new(ok:, next_step:, guidance: next_step.guidance, payload: ok ? {} : { errors: })
|
|
115
116
|
end
|
|
116
117
|
|
|
118
|
+
##
|
|
119
|
+
# @param test_result [Html2rss::Test::Result]
|
|
120
|
+
# @return [Outcome]
|
|
121
|
+
def test(test_result)
|
|
122
|
+
next_step = test_next_step(test_result)
|
|
123
|
+
new(
|
|
124
|
+
ok: test_result.success,
|
|
125
|
+
next_step:,
|
|
126
|
+
guidance: test_guidance(test_result, next_step),
|
|
127
|
+
payload: test_result.to_h
|
|
128
|
+
)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
##
|
|
132
|
+
# @param batch_result [Html2rss::Batch::BatchResult]
|
|
133
|
+
# @return [Outcome]
|
|
134
|
+
def batch_scrape(batch_result) = batch(batch_result, NextStep.scrape)
|
|
135
|
+
|
|
136
|
+
##
|
|
137
|
+
# @param batch_result [Html2rss::Batch::BatchResult]
|
|
138
|
+
# @return [Outcome]
|
|
139
|
+
def batch_inspect(batch_result) = batch(batch_result, NextStep.inspect)
|
|
140
|
+
|
|
141
|
+
##
|
|
142
|
+
# @param batch_result [Html2rss::Batch::BatchResult]
|
|
143
|
+
# @return [Outcome]
|
|
144
|
+
def batch_recon(batch_result) = batch(batch_result, NextStep.recon)
|
|
145
|
+
|
|
117
146
|
##
|
|
118
147
|
# @param rss [String]
|
|
119
148
|
# @param item_count [Integer]
|
|
120
149
|
# @param empty [Boolean] {FeedResult#empty?} (ship gate); defaults to zero items
|
|
150
|
+
# @param quality_report [Hash, nil] optional ship-quality audit summary
|
|
121
151
|
# @return [Outcome]
|
|
122
|
-
def apply(rss:, item_count:, empty: item_count.zero
|
|
152
|
+
def apply(rss:, item_count:, empty: item_count.zero?, quality_report: nil)
|
|
123
153
|
ok = !empty
|
|
124
|
-
next_step = ok ? NextStep.done : NextStep.
|
|
125
|
-
|
|
154
|
+
next_step = ok ? NextStep.done : NextStep.inspect
|
|
155
|
+
payload = { rss:, item_count: }
|
|
156
|
+
payload[:quality_report] = quality_report if quality_report
|
|
157
|
+
new(ok:, next_step:, guidance: next_step.guidance, payload:)
|
|
126
158
|
end
|
|
127
159
|
|
|
128
160
|
##
|
|
@@ -136,11 +168,16 @@ module Html2rss
|
|
|
136
168
|
|
|
137
169
|
private
|
|
138
170
|
|
|
171
|
+
def batch(batch_result, failure_step)
|
|
172
|
+
step = batch_result.successful.positive? ? NextStep.done : failure_step
|
|
173
|
+
new(ok: true, next_step: step, guidance: step.guidance, payload: batch_result.to_h)
|
|
174
|
+
end
|
|
175
|
+
|
|
139
176
|
def scrape_next_step(empty, botasaurus_configured:)
|
|
140
177
|
return NextStep.done unless empty
|
|
141
178
|
return NextStep.read_runtime unless botasaurus_configured
|
|
142
179
|
|
|
143
|
-
NextStep.
|
|
180
|
+
NextStep.inspect
|
|
144
181
|
end
|
|
145
182
|
|
|
146
183
|
def scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:)
|
|
@@ -150,22 +187,55 @@ module Html2rss
|
|
|
150
187
|
}
|
|
151
188
|
end
|
|
152
189
|
|
|
153
|
-
def inspect_next_step(
|
|
154
|
-
|
|
155
|
-
return NextStep.
|
|
156
|
-
return NextStep.capture_config if payload[:articles_count].to_i.positive?
|
|
190
|
+
def inspect_next_step(report)
|
|
191
|
+
return NextStep.recon if report.alternate_feeds?
|
|
192
|
+
return NextStep.capture if report.articles_count.positive?
|
|
157
193
|
|
|
158
|
-
NextStep.
|
|
194
|
+
NextStep.scrape
|
|
159
195
|
end
|
|
160
196
|
|
|
161
|
-
def
|
|
162
|
-
|
|
197
|
+
def recon_next_step(result)
|
|
198
|
+
return NextStep.done if result.defer?
|
|
199
|
+
return NextStep.capture if result.build?
|
|
200
|
+
|
|
201
|
+
NextStep.scrape
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def capture_next_step(articles_count:, has_selectors:, native_feed: nil)
|
|
205
|
+
return NextStep.done if native_feed
|
|
206
|
+
|
|
207
|
+
articles_count.positive? && has_selectors ? NextStep.test : NextStep.inspect
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def test_next_step(test_result)
|
|
211
|
+
return NextStep.apply if test_result.success
|
|
212
|
+
|
|
213
|
+
kind = test_result.failure_kind
|
|
214
|
+
return NextStep.validate if kind&.schema?
|
|
215
|
+
return NextStep.capture if kind&.execution? || kind&.min_items? || kind&.quality?
|
|
216
|
+
|
|
217
|
+
NextStep.capture
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def test_guidance(test_result, next_step)
|
|
221
|
+
base = if test_result.success
|
|
222
|
+
next_step.guidance
|
|
223
|
+
else
|
|
224
|
+
test_result.error_message || next_step.guidance
|
|
225
|
+
end
|
|
226
|
+
warnings = test_result.quality_report&.warnings
|
|
227
|
+
return base if warnings.nil? || warnings.empty?
|
|
228
|
+
|
|
229
|
+
"#{base} Review payload.quality_report warnings: #{warnings.join(', ')}."
|
|
163
230
|
end
|
|
164
231
|
|
|
165
232
|
def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
|
|
166
|
-
segment_strategy:, selected_strategy:, admission_drops:
|
|
233
|
+
segment_strategy:, selected_strategy:, admission_drops:, native_feed: nil,
|
|
234
|
+
suggested_channel_url: nil)
|
|
167
235
|
{
|
|
168
236
|
yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
|
|
237
|
+
**(native_feed ? { native_feed: native_feed.to_s } : {}),
|
|
238
|
+
**(suggested_channel_url ? { suggested_channel_url: suggested_channel_url.to_s } : {}),
|
|
169
239
|
**(segment_strategy ? { segment_strategy: segment_strategy.to_s } : {}),
|
|
170
240
|
**(selected_strategy ? { selected_strategy: selected_strategy.to_s } : {}),
|
|
171
241
|
**(admission_drops.any? ? { admission_drops: } : {})
|
|
@@ -175,14 +245,14 @@ module Html2rss
|
|
|
175
245
|
def next_step_for_error(error)
|
|
176
246
|
case error
|
|
177
247
|
when RequestService::BotasaurusConfigurationError then NextStep.read_runtime
|
|
178
|
-
when Contract::UnpublishedRequestError then NextStep.
|
|
248
|
+
when Contract::UnpublishedRequestError then NextStep.validate
|
|
179
249
|
when ArgumentError then argument_error_next_step(error)
|
|
180
|
-
else NextStep.
|
|
250
|
+
else NextStep.inspect
|
|
181
251
|
end
|
|
182
252
|
end
|
|
183
253
|
|
|
184
254
|
def argument_error_next_step(error)
|
|
185
|
-
XOR_ERROR.match?(error.message) ? NextStep.
|
|
255
|
+
XOR_ERROR.match?(error.message) ? NextStep.validate : NextStep.inspect
|
|
186
256
|
end
|
|
187
257
|
end
|
|
188
258
|
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
module MCP
|
|
5
|
+
##
|
|
6
|
+
# Runtime facts for the MCP process (env configuration, wire coercion).
|
|
7
|
+
module Runtime
|
|
8
|
+
# Published +html2rss://runtime+ resource (never leaks secrets).
|
|
9
|
+
Snapshot = Data.define(
|
|
10
|
+
:version,
|
|
11
|
+
:mcp_contract_version,
|
|
12
|
+
:catalog_fingerprint,
|
|
13
|
+
:tools,
|
|
14
|
+
:botasaurus_configured
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
##
|
|
20
|
+
# @return [Boolean] whether Botasaurus transport is configured in this process
|
|
21
|
+
def botasaurus_configured?
|
|
22
|
+
!ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
##
|
|
26
|
+
# @return [Snapshot] runtime capabilities and catalog identity for MCP clients
|
|
27
|
+
def snapshot
|
|
28
|
+
Snapshot.new(
|
|
29
|
+
version: Html2rss::VERSION,
|
|
30
|
+
mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
|
|
31
|
+
catalog_fingerprint: Contract.catalog_fingerprint,
|
|
32
|
+
tools: Contract.catalog_tools,
|
|
33
|
+
botasaurus_configured: botasaurus_configured?
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
##
|
|
38
|
+
# @param strategy [String, Symbol, nil]
|
|
39
|
+
# @return [Symbol]
|
|
40
|
+
def coerce_strategy(strategy)
|
|
41
|
+
(strategy || :auto).to_sym
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|