html2rss 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +82 -25
  3. data/html2rss.gemspec +0 -1
  4. data/lib/html2rss/article.rb +1 -2
  5. data/lib/html2rss/auto_source/README.md +8 -8
  6. data/lib/html2rss/auto_source/cleanup.rb +92 -1
  7. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
  8. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
  9. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
  10. data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
  11. data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
  12. data/lib/html2rss/auto_source/scraper.rb +2 -3
  13. data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
  14. data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
  15. data/lib/html2rss/auto_source/segmenter.rb +1 -2
  16. data/lib/html2rss/auto_source.rb +25 -11
  17. data/lib/html2rss/batch.rb +169 -0
  18. data/lib/html2rss/capture/README.md +15 -6
  19. data/lib/html2rss/capture.rb +158 -15
  20. data/lib/html2rss/channel.rb +1 -2
  21. data/lib/html2rss/cli/probe_view.rb +66 -0
  22. data/lib/html2rss/cli/render.rb +124 -0
  23. data/lib/html2rss/cli/validate.rb +91 -0
  24. data/lib/html2rss/cli.rb +318 -193
  25. data/lib/html2rss/config/request_headers.rb +4 -14
  26. data/lib/html2rss/config/schema.rb +7 -5
  27. data/lib/html2rss/config/validator.rb +1 -0
  28. data/lib/html2rss/config.rb +59 -22
  29. data/lib/html2rss/doctor/botasaurus.rb +130 -0
  30. data/lib/html2rss/feed_builder/rss.rb +1 -2
  31. data/lib/html2rss/feed_pipeline/README.md +5 -5
  32. data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
  33. data/lib/html2rss/feed_pipeline.rb +27 -16
  34. data/lib/html2rss/feed_resolution/README.md +11 -11
  35. data/lib/html2rss/feed_resolution.rb +6 -12
  36. data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
  37. data/lib/html2rss/html/article_extractor.rb +4 -9
  38. data/lib/html2rss/html/feed_link.rb +4 -2
  39. data/lib/html2rss/html/navigator.rb +4 -39
  40. data/lib/html2rss/html/probe.rb +82 -0
  41. data/lib/html2rss/html/sst_article_extractor.rb +1 -2
  42. data/lib/html2rss/link_destination/noise_policy.rb +1 -2
  43. data/lib/html2rss/link_destination/path_classifier.rb +20 -19
  44. data/lib/html2rss/mcp/README.md +80 -0
  45. data/lib/html2rss/mcp/contract.rb +135 -13
  46. data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
  47. data/lib/html2rss/mcp/outcome.rb +112 -42
  48. data/lib/html2rss/mcp/runtime.rb +45 -0
  49. data/lib/html2rss/mcp/server/tools.rb +277 -0
  50. data/lib/html2rss/mcp/server.rb +91 -232
  51. data/lib/html2rss/mcp.rb +3 -0
  52. data/lib/html2rss/page_recon/README.md +69 -0
  53. data/lib/html2rss/page_recon/diagnostics.rb +211 -0
  54. data/lib/html2rss/page_recon.rb +56 -2
  55. data/lib/html2rss/recon.rb +252 -0
  56. data/lib/html2rss/request_service/blocked_surface.rb +1 -0
  57. data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
  58. data/lib/html2rss/request_service/policy.rb +1 -2
  59. data/lib/html2rss/request_service/response.rb +3 -4
  60. data/lib/html2rss/request_service/strategy.rb +1 -2
  61. data/lib/html2rss/request_service.rb +2 -0
  62. data/lib/html2rss/request_session.rb +1 -2
  63. data/lib/html2rss/scrape_target.rb +1 -1
  64. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
  65. data/lib/html2rss/selectors/post_processors.rb +0 -1
  66. data/lib/html2rss/selectors.rb +33 -10
  67. data/lib/html2rss/sst/attrs.rb +1 -2
  68. data/lib/html2rss/sst/normalizer.rb +49 -11
  69. data/lib/html2rss/status.rb +3 -6
  70. data/lib/html2rss/syndication/README.md +6 -6
  71. data/lib/html2rss/syndication/discovery.rb +2 -4
  72. data/lib/html2rss/test/enhance_audit.rb +192 -0
  73. data/lib/html2rss/test.rb +458 -0
  74. data/lib/html2rss/url.rb +1 -2
  75. data/lib/html2rss/version.rb +1 -1
  76. data/lib/html2rss.rb +157 -35
  77. data/schema/html2rss-config.schema.json +9 -29
  78. metadata +17 -18
  79. data/lib/html2rss/mcp/inspect.rb +0 -138
  80. data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'digest'
4
+
3
5
  module Html2rss
4
6
  module MCP
5
7
  ##
@@ -9,6 +11,10 @@ module Html2rss
9
11
  # Published MCP request strategies (excludes +local_file+).
10
12
  STRATEGIES = %w[auto faraday botasaurus].freeze
11
13
 
14
+ # Bump when tool names, required inputs, or envelope semantics change (independent of gem +VERSION+).
15
+ MCP_CONTRACT_VERSION = 2
16
+ public_constant :MCP_CONTRACT_VERSION
17
+
12
18
  # Raised when apply/validate config uses an unpublished MCP request adapter.
13
19
  class UnpublishedRequestError < ArgumentError; end
14
20
 
@@ -51,14 +57,14 @@ module Html2rss
51
57
  }.freeze
52
58
  }.freeze
53
59
 
54
- # Input schema for +validate_config+ (config XOR yaml).
60
+ # Input schema for +validate+ (config XOR yaml).
55
61
  CONFIG_XOR_SCHEMA = {
56
62
  type: 'object',
57
63
  properties: CONFIG_XOR_PROPERTIES,
58
64
  oneOf: XOR_ONE_OF
59
65
  }.freeze
60
66
 
61
- # Input schema for +apply_config+ (required URL plus config XOR yaml).
67
+ # Input schema for +apply+ (required URL plus config XOR yaml).
62
68
  APPLY_INPUT_SCHEMA = {
63
69
  type: 'object',
64
70
  properties: { url: URL_PROPERTY, **CONFIG_XOR_PROPERTIES }.freeze,
@@ -66,7 +72,28 @@ module Html2rss
66
72
  oneOf: XOR_ONE_OF
67
73
  }.freeze
68
74
 
69
- # Input schema for +scrape_url+.
75
+ # Input schema for +test+.
76
+ TEST_INPUT_SCHEMA = {
77
+ type: 'object',
78
+ properties: {
79
+ **CONFIG_XOR_PROPERTIES,
80
+ min_items: { type: 'integer', description: 'Minimum required items (default: 1)', default: 1 },
81
+ strict_quality: {
82
+ type: 'boolean',
83
+ description: 'Fail when ship-quality audit thresholds are exceeded (default: false)',
84
+ default: false
85
+ },
86
+ compare_enhance: {
87
+ type: 'boolean',
88
+ description: 'Diagnostic: compare extraction with enhance on vs off on cached HTML (default: false)',
89
+ default: false
90
+ },
91
+ strategy: STRATEGY_PROPERTY
92
+ }.freeze,
93
+ oneOf: XOR_ONE_OF
94
+ }.freeze
95
+
96
+ # Input schema for +scrape+.
70
97
  SCRAPE_INPUT_SCHEMA = {
71
98
  type: 'object',
72
99
  properties: {
@@ -78,24 +105,87 @@ module Html2rss
78
105
  required: %w[url]
79
106
  }.freeze
80
107
 
81
- # Input schema for +inspect_url+.
108
+ # Input schema for +inspect+.
82
109
  INSPECT_INPUT_SCHEMA = {
83
110
  type: 'object',
84
111
  properties: { url: URL_PROPERTY, strategy: INSPECT_STRATEGY_PROPERTY }.freeze,
85
112
  required: %w[url]
86
113
  }.freeze
87
114
 
88
- # Input schema for +capture_config+.
115
+ # Input schema for +recon+.
116
+ RECON_INPUT_SCHEMA = INSPECT_INPUT_SCHEMA
117
+
118
+ # Input schema for +capture+.
89
119
  CAPTURE_INPUT_SCHEMA = {
90
120
  type: 'object',
91
121
  properties: {
92
122
  url: URL_PROPERTY,
93
123
  strategy: STRATEGY_PROPERTY,
94
- items_selector: { type: 'string', description: 'Optional CSS selector hint for items' }
124
+ items_selector: { type: 'string', description: 'Optional CSS selector hint for items' },
125
+ force: { type: 'boolean', description: 'Bypass native feed check', default: false },
126
+ topics: {
127
+ type: 'array',
128
+ items: { type: 'string' },
129
+ description: 'Directory topics override'
130
+ }.freeze,
131
+ title: { type: 'string', description: 'Channel title override' },
132
+ summary: { type: 'string', description: 'Directory summary override' },
133
+ enhance: { type: 'boolean', description: 'Force enhance on or off' },
134
+ limit: { type: 'integer', description: 'Max articles to keep' },
135
+ max_redirects: { type: 'integer', description: 'Optional redirect limit override' },
136
+ max_requests: { type: 'integer', description: 'Optional request budget override' }
95
137
  }.freeze,
96
138
  required: %w[url]
97
139
  }.freeze
98
140
 
141
+ ##
142
+ # Shared JSON Schema for batch URL tools (+batch_scrape+, +batch_inspect+, +batch_recon+).
143
+ #
144
+ # @param urls_description [String] description for the +urls+ array property
145
+ # @param strategy_property [Hash] strategy JSON Schema property
146
+ # @param extra_properties [Hash] additional tool-specific properties (e.g. +limit+ on scrape)
147
+ # @return [Hash]
148
+ def self.batch_urls_input_schema(urls_description:, strategy_property:, extra_properties: {}) # rubocop:disable Metrics/MethodLength
149
+ {
150
+ type: 'object',
151
+ properties: {
152
+ urls: {
153
+ type: 'array',
154
+ items: URL_PROPERTY,
155
+ minItems: 1,
156
+ maxItems: 25,
157
+ description: urls_description
158
+ }.freeze,
159
+ strategy: strategy_property,
160
+ concurrency: {
161
+ type: 'integer',
162
+ description: 'Max parallel worker threads (1..10, default: 5)',
163
+ default: 5
164
+ },
165
+ **extra_properties
166
+ }.freeze,
167
+ required: %w[urls]
168
+ }
169
+ end
170
+
171
+ # Input schema for +batch_scrape+.
172
+ BATCH_SCRAPE_INPUT_SCHEMA = batch_urls_input_schema(
173
+ urls_description: 'List of page URLs to scrape (1..25)',
174
+ strategy_property: STRATEGY_PROPERTY,
175
+ extra_properties: {
176
+ limit: { type: 'integer', description: 'Max articles per URL to keep (default 10)', default: 10 }
177
+ }
178
+ ).freeze
179
+
180
+ # Input schema for +batch_inspect+.
181
+ BATCH_INSPECT_INPUT_SCHEMA = batch_urls_input_schema(
182
+ urls_description: 'List of page URLs to inspect (1..25)',
183
+ strategy_property: INSPECT_STRATEGY_PROPERTY
184
+ ).freeze
185
+
186
+ # Input schema for +batch_recon+.
187
+ BATCH_RECON_INPUT_SCHEMA = BATCH_INSPECT_INPUT_SCHEMA
188
+
99
189
  # Tool annotations for open-world read-only tools.
100
190
  ANNOTATIONS_OPEN_WORLD = {
101
191
  read_only_hint: true,
@@ -104,19 +194,51 @@ module Html2rss
104
194
  open_world_hint: true
105
195
  }.freeze
106
196
 
107
- # Tool annotations for +validate_config+ (closed world).
197
+ # Tool annotations for +validate+ (closed world).
108
198
  ANNOTATIONS_VALIDATE = ANNOTATIONS_OPEN_WORLD.merge(open_world_hint: false).freeze
109
199
 
110
200
  # Human titles for +tools/list+.
111
201
  TITLES = {
112
- scrape_url: 'Scrape URL',
113
- inspect_url: 'Inspect URL',
114
- capture_config: 'Capture feed config',
115
- validate_config: 'Validate feed config',
116
- apply_config: 'Apply feed config'
202
+ scrape: 'Scrape',
203
+ inspect: 'Inspect',
204
+ recon: 'Recon',
205
+ batch_scrape: 'Batch scrape',
206
+ batch_inspect: 'Batch inspect',
207
+ batch_recon: 'Batch recon',
208
+ capture: 'Capture',
209
+ validate: 'Validate',
210
+ apply: 'Apply',
211
+ test: 'Test'
117
212
  }.freeze
118
213
 
214
+ # @api private
215
+ CATALOG_ENTRY_LINE = lambda do |entry|
216
+ schema = entry.fetch(:input_schema)
217
+ required = Array(schema[:required]).sort.join(',')
218
+ one_of = Array(schema[:oneOf]).map { |branch| Array(branch[:required]).sort.join('+') }.sort.join('|')
219
+ [entry.fetch(:name), required, one_of].reject(&:empty?).join(':')
220
+ end.freeze
221
+
119
222
  class << self
223
+ ##
224
+ # Canonical MCP tool names in alphabetical order (same set as +tools/list+).
225
+ #
226
+ # @return [Array<String>]
227
+ def catalog_tools
228
+ Server::Tools::TOOLS.map { |entry| entry.fetch(:name) }.sort
229
+ end
230
+
231
+ ##
232
+ # Stable fingerprint of published tools, required keys, and +oneOf+ branches.
233
+ # Clients compare against a cached +tools/list+ to detect stale catalogs.
234
+ # Bump {MCP_CONTRACT_VERSION} for envelope or breaking wire semantics only.
235
+ #
236
+ # @return [String] 16-char hex digest prefix
237
+ def catalog_fingerprint
238
+ lines = Server::Tools::TOOLS.sort_by { |entry| entry.fetch(:name) }.map(&CATALOG_ENTRY_LINE)
239
+ Digest::SHA256.hexdigest(lines.join("\n")).slice(0, 16)
240
+ end
241
+
120
242
  ##
121
243
  # Envelope JSON Schema. Built lazily so Zeitwerk can load Contract before Outcome.
122
244
  #
@@ -151,7 +273,7 @@ module Html2rss
151
273
 
152
274
  ##
153
275
  # Rejects unpublished MCP request adapters so apply/validate cannot
154
- # {File.read} arbitrary paths. CLI and Config still allow +local_file+.
276
+ # +File.read+ arbitrary paths. CLI and Config still allow +local_file+.
155
277
  #
156
278
  # @param config [Hash]
157
279
  # @return [void]
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ module MCP
5
+ class Outcome
6
+ ##
7
+ # Single source of truth for MCP agent instructions, next-step guidance,
8
+ # and prompt bodies. {Server} delegates here — do not duplicate prose in
9
+ # +server.rb+.
10
+ class Playbook
11
+ # Default guidance copy keyed by {Outcome::NextStep::NAMES}.
12
+ GUIDANCE = {
13
+ done: 'Done. Read payload for the result.',
14
+ inspect: 'Call inspect next. Read payload for diagnostics (final_url, status, ' \
15
+ 'scheme_downgrade, alternate_feeds, likely_js_shell, redirect_summary).',
16
+ recon: 'Call recon next. Read payload for verdict and native_feed preference.',
17
+ validate: 'Call validate with payload.yaml or a config hash (XOR, not both).',
18
+ apply: 'Call apply next. Confirm payload.item_count before shipping.',
19
+ scrape: 'Call scrape for articles now. strategy auto already runs Faraday then Botasaurus ' \
20
+ 'and promotes native RSS/Atom when present.',
21
+ capture: 'Call capture for a reusable YAML draft, then follow next_step.',
22
+ read_runtime: 'Read html2rss://runtime. Compare mcp_contract_version and catalog_fingerprint ' \
23
+ 'to your cached tools/list before retrying unknown tools. ' \
24
+ 'Set BOTASAURUS_SCRAPER_URL on the MCP process if botasaurus_configured is false.',
25
+ test: 'Call test next (schema + live extraction). Confirm payload.item_count, ' \
26
+ 'failure_kind, and payload.quality_report warnings before shipping.'
27
+ }.freeze
28
+
29
+ class << self
30
+ ##
31
+ # Published MCP server instructions (decision tree for agents).
32
+ #
33
+ # @return [String]
34
+ def instructions # rubocop:disable Metrics/MethodLength -- agent decision tree is the published contract
35
+ <<~TEXT.strip
36
+ html2rss MCP — decide which tool to call:
37
+
38
+ 1. Need articles now (no saved config)? → scrape (or batch_scrape for multiple)
39
+ - strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
40
+ - Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
41
+ 2. Need a reusable feed YAML? → capture → test → apply
42
+ - capture returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Default enhance follows capture evidence (false when admission_drops show chrome); override only when needed.
43
+ - test runs schema + live extraction (min items). apply is the ship gate (isError on zero items). Confirm payload.item_count and payload.quality_report warnings (including enhance_gains when selectors.items.enhance is true). Use compare_enhance on test for enhance on/off diagnostics.
44
+ - validate alone is for schema-only checks; on success next_step is test.
45
+ 3. Weak scrape/capture or recon (final URL, status, https→http, rel=alternate feeds)? → inspect (or batch_inspect). Read likely_js_shell vs blocked_surface when articles_count is 0. When alternates warrant it, inspect next_step is recon.
46
+ 4. Have a config already? → validate (must succeed) → test → apply
47
+ 5. Schema / extractors / strategies / runtime → resources html2rss://schema|extractors|strategies|runtime
48
+ - runtime publishes version, mcp_contract_version, catalog_fingerprint, tools, botasaurus_configured.
49
+ - Refresh tools/list when catalog_fingerprint differs from your cache.
50
+
51
+ Prefer capture for durable config; scrape / batch_scrape for one-shot extraction.
52
+ Follow envelope next_step and guidance. Botasaurus needs BOTASAURUS_SCRAPER_URL in this process env (read html2rss://runtime; the URL is never returned).
53
+ TEXT
54
+ end
55
+
56
+ ##
57
+ # @param url [String]
58
+ # @return [String]
59
+ def scrape_webpage_prompt(url)
60
+ <<~MSG.strip
61
+ Scrape #{url} with scrape (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
62
+ Follow envelope next_step and guidance. Call inspect only if articles are empty/weak or you need diagnostics (final_url, status, scheme_downgrade, alternate_feeds). When inspect finds alternates, follow next_step to recon.
63
+ Do not retry scrape with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
64
+ Return payload.items (not a raw JSON array).
65
+ MSG
66
+ end
67
+
68
+ ##
69
+ # @param url [String]
70
+ # @return [String]
71
+ def capture_feed_config_prompt(url)
72
+ <<~MSG.strip
73
+ Build a reusable html2rss feed config for #{url}:
74
+ 1) capture — YAML is payload.yaml. Check payload.articles_count, payload.has_selectors, and payload.suggested_channel_url. enhance defaults from admission evidence (false when chrome drops are high). When payload.native_feed is set, follow next_step (done — use the native feed).
75
+ 2) Follow next_step. If weak or you need recon, inspect then recon when alternates warrant it. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
76
+ 3) test with yaml (or config hash) — schema + live extraction. On :schema failure, validate; on :execution/:min_items, recapture. Read payload.quality_report.enhance_gains when enhance is on; optional compare_enhance compares enhance off vs on without changing shipped RSS.
77
+ 4) apply — isError if zero items. Confirm payload.item_count and payload.quality_report (including enhance_gains) before shipping.
78
+ If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
79
+ MSG
80
+ end
81
+
82
+ ##
83
+ # @param report [PageRecon::Diagnostics::Report]
84
+ # @return [String]
85
+ def inspect_guidance(report)
86
+ return GUIDANCE.fetch(:inspect) unless report.articles_count.zero?
87
+
88
+ empty_extract_guidance(report.data)
89
+ end
90
+
91
+ ##
92
+ # @param data [Hash{Symbol => Object}]
93
+ # @return [String]
94
+ def empty_extract_guidance(data) # rubocop:disable Metrics/MethodLength
95
+ if data[:blocked_surface] || data[:surface_category].to_s == 'blocked_surface'
96
+ return 'Blocked or anti-bot interstitial likely. Retry scrape with strategy botasaurus once ' \
97
+ '(or CLI inspect --deep when BOTASAURUS_SCRAPER_URL is set). ' \
98
+ 'Do not retry explicit faraday after auto.'
99
+ end
100
+ if data[:likely_js_shell]
101
+ return 'JS-rendered shell likely (html_present, zero articles). Use strategy auto or botasaurus; ' \
102
+ 'CLI inspect --deep for one Botasaurus diagnostic hop.'
103
+ end
104
+
105
+ 'Empty extract on a static-looking page. Verify redirect_summary.final_url and surface; ' \
106
+ 'capture may need selector hints.'
107
+ end
108
+
109
+ ##
110
+ # @param result [Html2rss::Recon::Result]
111
+ # @param next_step [Outcome::NextStep]
112
+ # @return [String]
113
+ def recon_guidance(result, next_step)
114
+ return next_step.guidance unless result.scheme_downgrade
115
+
116
+ "#{next_step.guidance} HTTPS→HTTP downgrade detected: try one Botasaurus scrape before DROP."
117
+ end
118
+ end
119
+ end
120
+ end
121
+ end
122
+ end
@@ -5,10 +5,10 @@ module Html2rss
5
5
  Outcome = Data.define(:ok, :next_step, :guidance, :payload)
6
6
 
7
7
  ##
8
- # Typed MCP tool result. Owns next-step policy and guidance copy so the
9
- # protocol adapter does not branch on quality heuristics.
10
- class Outcome
11
- # Matches {ConfigArgument} XOR {ArgumentError} messages.
8
+ # Typed MCP tool result. Owns next-step policy; guidance copy lives in
9
+ # {Playbook}.
10
+ class Outcome # rubocop:disable Metrics/ClassLength -- next-step policy + factories stay co-located
11
+ # Matches {ConfigArgument} XOR +ArgumentError+ messages.
12
12
  XOR_ERROR = /exactly one of config or yaml/
13
13
  NextStep = Data.define(:name, :guidance)
14
14
 
@@ -16,20 +16,7 @@ module Html2rss
16
16
  # Closed set of agent next actions. Invalid names cannot be constructed.
17
17
  class NextStep
18
18
  # Wire names for +next_step+.
19
- NAMES = %i[done inspect_url validate_config apply_config scrape_url capture_config read_runtime].freeze
20
- # Default guidance copy keyed by {NAMES}.
21
- GUIDANCE = {
22
- done: 'Done. Read payload for the result.',
23
- inspect_url: 'Call inspect_url next. Read payload for recon (final_url, status, ' \
24
- 'scheme_downgrade, alternate_feeds).',
25
- validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
26
- apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
27
- scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus ' \
28
- 'and promotes native RSS/Atom when present.',
29
- capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
30
- read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
31
- 'if botasaurus_configured is false.'
32
- }.freeze
19
+ NAMES = %i[done inspect recon validate apply scrape capture read_runtime test].freeze
33
20
 
34
21
  ##
35
22
  # @param name [Symbol, String]
@@ -38,7 +25,7 @@ module Html2rss
38
25
  step = name.to_sym
39
26
  raise ArgumentError, "unknown next_step: #{name.inspect}" unless NAMES.include?(step)
40
27
 
41
- super(name: step, guidance: (guidance || GUIDANCE.fetch(step)).freeze)
28
+ super(name: step, guidance: (guidance || Playbook::GUIDANCE.fetch(step)).freeze)
42
29
  end
43
30
 
44
31
  class << self
@@ -79,11 +66,21 @@ module Html2rss
79
66
  end
80
67
 
81
68
  ##
82
- # @param payload [Hash] inspect recon Hash
69
+ # @param report [PageRecon::Diagnostics::Report]
83
70
  # @return [Outcome]
84
- def inspect(payload:)
85
- next_step = inspect_next_step(payload)
86
- new(ok: true, next_step:, guidance: next_step.guidance, payload:)
71
+ def inspect(report:)
72
+ next_step = inspect_next_step(report)
73
+ guidance = Playbook.inspect_guidance(report)
74
+ new(ok: true, next_step:, guidance:, payload: report.to_wire_h)
75
+ end
76
+
77
+ ##
78
+ # @param result [Html2rss::Recon::Result]
79
+ # @return [Outcome]
80
+ def recon(result:)
81
+ next_step = recon_next_step(result)
82
+ guidance = Playbook.recon_guidance(result, next_step)
83
+ new(ok: true, next_step:, guidance:, payload: result.to_h)
87
84
  end
88
85
 
89
86
  ##
@@ -95,13 +92,17 @@ module Html2rss
95
92
  # @param segment_strategy [Symbol, String, nil]
96
93
  # @param selected_strategy [Symbol, String, nil]
97
94
  # @param admission_drops [Hash]
95
+ # @param native_feed [String, nil]
96
+ # @param suggested_channel_url [String, nil]
98
97
  # @return [Outcome]
99
98
  def capture(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
100
- segment_strategy: nil, selected_strategy: nil, admission_drops: {})
101
- next_step = capture_next_step(articles_count:, has_selectors:)
99
+ segment_strategy: nil, selected_strategy: nil, admission_drops: {}, native_feed: nil,
100
+ suggested_channel_url: nil)
101
+ next_step = capture_next_step(articles_count:, has_selectors:, native_feed:)
102
102
  new(ok: true, next_step:, guidance: next_step.guidance, payload: capture_payload(
103
103
  yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:,
104
- segment_strategy:, selected_strategy:, admission_drops:
104
+ segment_strategy:, selected_strategy:, admission_drops:, native_feed:,
105
+ suggested_channel_url:
105
106
  ))
106
107
  end
107
108
 
@@ -110,19 +111,50 @@ module Html2rss
110
111
  # @return [Outcome]
111
112
  def validate(errors:)
112
113
  ok = errors.nil?
113
- next_step = ok ? NextStep.apply_config : NextStep.validate_config
114
+ next_step = ok ? NextStep.test : NextStep.validate
114
115
  new(ok:, next_step:, guidance: next_step.guidance, payload: ok ? {} : { errors: })
115
116
  end
116
117
 
118
+ ##
119
+ # @param test_result [Html2rss::Test::Result]
120
+ # @return [Outcome]
121
+ def test(test_result)
122
+ next_step = test_next_step(test_result)
123
+ new(
124
+ ok: test_result.success,
125
+ next_step:,
126
+ guidance: test_guidance(test_result, next_step),
127
+ payload: test_result.to_h
128
+ )
129
+ end
130
+
131
+ ##
132
+ # @param batch_result [Html2rss::Batch::BatchResult]
133
+ # @return [Outcome]
134
+ def batch_scrape(batch_result) = batch(batch_result, NextStep.scrape)
135
+
136
+ ##
137
+ # @param batch_result [Html2rss::Batch::BatchResult]
138
+ # @return [Outcome]
139
+ def batch_inspect(batch_result) = batch(batch_result, NextStep.inspect)
140
+
141
+ ##
142
+ # @param batch_result [Html2rss::Batch::BatchResult]
143
+ # @return [Outcome]
144
+ def batch_recon(batch_result) = batch(batch_result, NextStep.recon)
145
+
117
146
  ##
118
147
  # @param rss [String]
119
148
  # @param item_count [Integer]
120
149
  # @param empty [Boolean] {FeedResult#empty?} (ship gate); defaults to zero items
150
+ # @param quality_report [Hash, nil] optional ship-quality audit summary
121
151
  # @return [Outcome]
122
- def apply(rss:, item_count:, empty: item_count.zero?)
152
+ def apply(rss:, item_count:, empty: item_count.zero?, quality_report: nil)
123
153
  ok = !empty
124
- next_step = ok ? NextStep.done : NextStep.inspect_url
125
- new(ok:, next_step:, guidance: next_step.guidance, payload: { rss:, item_count: })
154
+ next_step = ok ? NextStep.done : NextStep.inspect
155
+ payload = { rss:, item_count: }
156
+ payload[:quality_report] = quality_report if quality_report
157
+ new(ok:, next_step:, guidance: next_step.guidance, payload:)
126
158
  end
127
159
 
128
160
  ##
@@ -136,11 +168,16 @@ module Html2rss
136
168
 
137
169
  private
138
170
 
171
+ def batch(batch_result, failure_step)
172
+ step = batch_result.successful.positive? ? NextStep.done : failure_step
173
+ new(ok: true, next_step: step, guidance: step.guidance, payload: batch_result.to_h)
174
+ end
175
+
139
176
  def scrape_next_step(empty, botasaurus_configured:)
140
177
  return NextStep.done unless empty
141
178
  return NextStep.read_runtime unless botasaurus_configured
142
179
 
143
- NextStep.inspect_url
180
+ NextStep.inspect
144
181
  end
145
182
 
146
183
  def scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:)
@@ -150,22 +187,55 @@ module Html2rss
150
187
  }
151
188
  end
152
189
 
153
- def inspect_next_step(payload)
154
- # Runtime scrape_url now consumes native alternates (NativeFeed / direct feed parse).
155
- return NextStep.scrape_url if Array(payload[:alternate_feeds]).any?
156
- return NextStep.capture_config if payload[:articles_count].to_i.positive?
190
+ def inspect_next_step(report)
191
+ return NextStep.recon if report.alternate_feeds?
192
+ return NextStep.capture if report.articles_count.positive?
157
193
 
158
- NextStep.scrape_url
194
+ NextStep.scrape
159
195
  end
160
196
 
161
- def capture_next_step(articles_count:, has_selectors:)
162
- articles_count.positive? && has_selectors ? NextStep.validate_config : NextStep.inspect_url
197
+ def recon_next_step(result)
198
+ return NextStep.done if result.defer?
199
+ return NextStep.capture if result.build?
200
+
201
+ NextStep.scrape
202
+ end
203
+
204
+ def capture_next_step(articles_count:, has_selectors:, native_feed: nil)
205
+ return NextStep.done if native_feed
206
+
207
+ articles_count.positive? && has_selectors ? NextStep.test : NextStep.inspect
208
+ end
209
+
210
+ def test_next_step(test_result)
211
+ return NextStep.apply if test_result.success
212
+
213
+ kind = test_result.failure_kind
214
+ return NextStep.validate if kind&.schema?
215
+ return NextStep.capture if kind&.execution? || kind&.min_items? || kind&.quality?
216
+
217
+ NextStep.capture
218
+ end
219
+
220
+ def test_guidance(test_result, next_step)
221
+ base = if test_result.success
222
+ next_step.guidance
223
+ else
224
+ test_result.error_message || next_step.guidance
225
+ end
226
+ warnings = test_result.quality_report&.warnings
227
+ return base if warnings.nil? || warnings.empty?
228
+
229
+ "#{base} Review payload.quality_report warnings: #{warnings.join(', ')}."
163
230
  end
164
231
 
165
232
  def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
166
- segment_strategy:, selected_strategy:, admission_drops:)
233
+ segment_strategy:, selected_strategy:, admission_drops:, native_feed: nil,
234
+ suggested_channel_url: nil)
167
235
  {
168
236
  yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
237
+ **(native_feed ? { native_feed: native_feed.to_s } : {}),
238
+ **(suggested_channel_url ? { suggested_channel_url: suggested_channel_url.to_s } : {}),
169
239
  **(segment_strategy ? { segment_strategy: segment_strategy.to_s } : {}),
170
240
  **(selected_strategy ? { selected_strategy: selected_strategy.to_s } : {}),
171
241
  **(admission_drops.any? ? { admission_drops: } : {})
@@ -175,14 +245,14 @@ module Html2rss
175
245
  def next_step_for_error(error)
176
246
  case error
177
247
  when RequestService::BotasaurusConfigurationError then NextStep.read_runtime
178
- when Contract::UnpublishedRequestError then NextStep.validate_config
248
+ when Contract::UnpublishedRequestError then NextStep.validate
179
249
  when ArgumentError then argument_error_next_step(error)
180
- else NextStep.inspect_url
250
+ else NextStep.inspect
181
251
  end
182
252
  end
183
253
 
184
254
  def argument_error_next_step(error)
185
- XOR_ERROR.match?(error.message) ? NextStep.validate_config : NextStep.inspect_url
255
+ XOR_ERROR.match?(error.message) ? NextStep.validate : NextStep.inspect
186
256
  end
187
257
  end
188
258
  end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ module MCP
5
+ ##
6
+ # Runtime facts for the MCP process (env configuration, wire coercion).
7
+ module Runtime
8
+ # Published +html2rss://runtime+ resource (never leaks secrets).
9
+ Snapshot = Data.define(
10
+ :version,
11
+ :mcp_contract_version,
12
+ :catalog_fingerprint,
13
+ :tools,
14
+ :botasaurus_configured
15
+ )
16
+
17
+ module_function
18
+
19
+ ##
20
+ # @return [Boolean] whether Botasaurus transport is configured in this process
21
+ def botasaurus_configured?
22
+ !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
23
+ end
24
+
25
+ ##
26
+ # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
27
+ def snapshot
28
+ Snapshot.new(
29
+ version: Html2rss::VERSION,
30
+ mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
31
+ catalog_fingerprint: Contract.catalog_fingerprint,
32
+ tools: Contract.catalog_tools,
33
+ botasaurus_configured: botasaurus_configured?
34
+ )
35
+ end
36
+
37
+ ##
38
+ # @param strategy [String, Symbol, nil]
39
+ # @return [Symbol]
40
+ def coerce_strategy(strategy)
41
+ (strategy || :auto).to_sym
42
+ end
43
+ end
44
+ end
45
+ end