html2rss 0.32.4 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. data/lib/html2rss/article/enclosure.rb +2 -6
  3. data/lib/html2rss/auto_source/cleanup.rb +1 -3
  4. data/lib/html2rss/auto_source/scraper/html.rb +2 -8
  5. data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
  6. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
  7. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
  8. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  9. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
  10. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
  11. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
  12. data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
  13. data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
  14. data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
  15. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  16. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
  17. data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
  18. data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
  19. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
  20. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  21. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
  22. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  23. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
  25. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
  26. data/lib/html2rss/auto_source/scraper.rb +0 -11
  27. data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
  28. data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
  29. data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
  30. data/lib/html2rss/auto_source.rb +1 -1
  31. data/lib/html2rss/batch.rb +22 -32
  32. data/lib/html2rss/capture/README.md +1 -1
  33. data/lib/html2rss/capture.rb +7 -27
  34. data/lib/html2rss/cli/render.rb +97 -96
  35. data/lib/html2rss/cli/validate.rb +2 -4
  36. data/lib/html2rss/cli.rb +33 -27
  37. data/lib/html2rss/config/request_controls.rb +10 -2
  38. data/lib/html2rss/config/schema.rb +8 -10
  39. data/lib/html2rss/config/selectors_validator.rb +1 -0
  40. data/lib/html2rss/config/validator.rb +3 -3
  41. data/lib/html2rss/config.rb +1 -1
  42. data/lib/html2rss/defaults.rb +5 -16
  43. data/lib/html2rss/doctor/botasaurus.rb +11 -19
  44. data/lib/html2rss/error.rb +7 -3
  45. data/lib/html2rss/feed_builder/rss.rb +1 -1
  46. data/lib/html2rss/feed_pipeline/auto_fallback.rb +3 -3
  47. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  48. data/lib/html2rss/feed_resolution/options.rb +3 -7
  49. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  50. data/lib/html2rss/feed_resolution/probe.rb +3 -3
  51. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  52. data/lib/html2rss/feed_resolution.rb +5 -9
  53. data/lib/html2rss/feed_result.rb +4 -3
  54. data/lib/html2rss/hash_util.rb +91 -90
  55. data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
  56. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  57. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
  58. data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
  59. data/lib/html2rss/html/probe.rb +51 -51
  60. data/lib/html2rss/link_destination/path_classifier.rb +11 -33
  61. data/lib/html2rss/mcp/config_argument.rb +9 -2
  62. data/lib/html2rss/mcp/contract.rb +7 -5
  63. data/lib/html2rss/mcp/outcome.rb +4 -5
  64. data/lib/html2rss/mcp/runtime.rb +17 -24
  65. data/lib/html2rss/mcp/server/tools.rb +9 -9
  66. data/lib/html2rss/option.rb +48 -0
  67. data/lib/html2rss/page_recon/diagnostics.rb +7 -9
  68. data/lib/html2rss/page_recon.rb +0 -4
  69. data/lib/html2rss/recon.rb +10 -12
  70. data/lib/html2rss/request_service/blocked_surface.rb +3 -7
  71. data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
  72. data/lib/html2rss/request_service/budget.rb +3 -24
  73. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  74. data/lib/html2rss/request_service/context.rb +1 -3
  75. data/lib/html2rss/request_service/local_file_strategy.rb +0 -1
  76. data/lib/html2rss/request_service/network_guard.rb +3 -2
  77. data/lib/html2rss/request_service/policy.rb +8 -26
  78. data/lib/html2rss/request_service/response.rb +3 -6
  79. data/lib/html2rss/request_session/pager/base.rb +3 -3
  80. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  81. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  82. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  83. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  84. data/lib/html2rss/scoring/engine.rb +4 -32
  85. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  86. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  87. data/lib/html2rss/selectors/json_xml.rb +33 -34
  88. data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
  89. data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
  90. data/lib/html2rss/selectors/schema_export.rb +60 -60
  91. data/lib/html2rss/selectors.rb +14 -18
  92. data/lib/html2rss/sst/node.rb +9 -3
  93. data/lib/html2rss/sst/normalizer.rb +25 -24
  94. data/lib/html2rss/sst/text.rb +53 -57
  95. data/lib/html2rss/status.rb +23 -30
  96. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  97. data/lib/html2rss/syndication/discovery.rb +117 -187
  98. data/lib/html2rss/syndication/parser.rb +92 -115
  99. data/lib/html2rss/test/enhance_audit.rb +13 -15
  100. data/lib/html2rss/test/policy.rb +9 -11
  101. data/lib/html2rss/test.rb +29 -48
  102. data/lib/html2rss/url.rb +53 -53
  103. data/lib/html2rss/version.rb +1 -1
  104. data/lib/html2rss.rb +2 -1
  105. metadata +4 -2
@@ -10,116 +10,117 @@ module Html2rss
10
10
  # ANSI colors for recon verdict labels in text output.
11
11
  VERDICT_COLORS = { build: "\e[32m", defer: "\e[33m" }.freeze
12
12
 
13
- module_function
14
-
15
- ##
16
- # @param results [Array<Hash>]
17
- # @param format [String]
18
- # @param batch_mode [Boolean]
19
- # @return [void]
20
- def inspect_output(results, format:, batch_mode:)
21
- if format == 'json'
22
- json(results, batch_mode)
23
- else
24
- results.each { |data| probe_card(ProbeView.from_wire(data)) }
13
+ class << self
14
+ ##
15
+ # @param results [Array<Hash>]
16
+ # @param format [String]
17
+ # @param batch_mode [Boolean]
18
+ # @return [void]
19
+ def inspect_output(results, format:, batch_mode:)
20
+ if format == 'json'
21
+ json(results, batch_mode)
22
+ else
23
+ results.each { |data| probe_card(ProbeView.from_wire(data)) }
24
+ end
25
25
  end
26
- end
27
26
 
28
- ##
29
- # @param results [Array<Html2rss::Recon::Result>]
30
- # @param format [String]
31
- # @param batch_mode [Boolean]
32
- # @param url_only [Boolean]
33
- # @return [void]
34
- def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
35
- if url_only
36
- results.each { |r| puts r.requested_url }
37
- elsif format == 'json'
38
- json(results.map(&:to_h), batch_mode)
39
- elsif format == 'tsv'
40
- puts %w[verdict status requested_url final_url native_feed notes].join("\t")
41
- results.each do |r|
42
- row = [
43
- r.verdict.to_s.upcase,
44
- r.status || '-',
45
- r.requested_url,
46
- r.final_url,
47
- r.native_feed || '-',
48
- r.notes.join('; ')
49
- ]
50
- puts row.join("\t")
27
+ ##
28
+ # @param results [Array<Html2rss::Recon::Result>]
29
+ # @param format [String]
30
+ # @param batch_mode [Boolean]
31
+ # @param url_only [Boolean]
32
+ # @return [void]
33
+ def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
34
+ if url_only
35
+ results.each { |r| puts r.requested_url }
36
+ elsif format == 'json'
37
+ json(results.map(&:to_h), batch_mode)
38
+ elsif format == 'tsv'
39
+ puts %w[verdict status requested_url final_url native_feed notes].join("\t")
40
+ results.each do |r|
41
+ row = [
42
+ r.verdict.to_s.upcase,
43
+ r.status || '-',
44
+ r.requested_url,
45
+ r.final_url,
46
+ r.native_feed || '-',
47
+ r.notes.join('; ')
48
+ ]
49
+ puts row.join("\t")
50
+ end
51
+ else
52
+ results.each { |r| probe_card(ProbeView.from_recon(r)) }
51
53
  end
52
- else
53
- results.each { |r| probe_card(ProbeView.from_recon(r)) }
54
54
  end
55
- end
56
55
 
57
- ##
58
- # @param view [ProbeView]
59
- # @return [void]
60
- def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
61
- if view.verdict
62
- color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
63
- puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
64
- else
65
- puts view.requested
66
- end
67
- probe_lines(view)
68
- if view.alternate_feeds.any?
69
- hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
70
- puts " Feeds: #{hrefs.join(', ')}" if hrefs.any?
56
+ ##
57
+ # @param view [ProbeView]
58
+ # @return [void]
59
+ def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
60
+ if view.verdict
61
+ color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
62
+ puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
63
+ else
64
+ puts view.requested
65
+ end
66
+ probe_lines(view)
67
+ if view.alternate_feeds.any?
68
+ hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
69
+ puts " Feeds: #{hrefs.join(', ')}" if hrefs.any?
70
+ end
71
+ puts " Feed: #{view.native_feed}" if view.native_feed
72
+ puts " Notes: #{view.notes.join(', ')}" if view.notes.any?
73
+ puts " Strategy: #{view.strategy}" if view.strategy
74
+ puts ''
71
75
  end
72
- puts " Feed: #{view.native_feed}" if view.native_feed
73
- puts " Notes: #{view.notes.join(', ')}" if view.notes.any?
74
- puts " Strategy: #{view.strategy}" if view.strategy
75
- puts ''
76
- end
77
76
 
78
- ##
79
- # @param result [Html2rss::Test::Result]
80
- # @param source [String]
81
- # @return [void]
82
- def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
83
- if result.success
84
- puts "\e[32m✓ Schema valid\e[0m (#{source})"
85
- dur = "#{result.duration_seconds}s"
86
- puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
87
- puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
88
- if result.sample_items.any?
89
- puts 'Sample items:'
90
- result.sample_items.each_with_index do |item, i|
91
- puts " #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
92
- puts " #{item[:url]}"
77
+ ##
78
+ # @param result [Html2rss::Test::Result]
79
+ # @param source [String]
80
+ # @return [void]
81
+ def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
82
+ if result.success
83
+ puts "\e[32m✓ Schema valid\e[0m (#{source})"
84
+ dur = "#{result.duration_seconds}s"
85
+ puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
86
+ puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
87
+ if result.sample_items.any?
88
+ puts 'Sample items:'
89
+ result.sample_items.each_with_index do |item, i|
90
+ puts " #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
91
+ puts " #{item[:url]}"
92
+ end
93
+ end
94
+ else
95
+ $stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
96
+ $stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
97
+ result.validation_issues&.each do |issue|
98
+ $stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
93
99
  end
94
- end
95
- else
96
- $stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
97
- $stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
98
- result.validation_issues&.each do |issue|
99
- $stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
100
100
  end
101
101
  end
102
- end
103
102
 
104
- ##
105
- # @param data [Object]
106
- # @param batch_mode [Boolean]
107
- # @return [void]
108
- def json(data, batch_mode)
109
- payload = batch_mode ? data : data.first
110
- puts JSON.pretty_generate(payload)
111
- end
103
+ ##
104
+ # @param data [Object]
105
+ # @param batch_mode [Boolean]
106
+ # @return [void]
107
+ def json(data, batch_mode)
108
+ payload = batch_mode ? data : data.first
109
+ puts JSON.pretty_generate(payload)
110
+ end
112
111
 
113
- ##
114
- # @param view [ProbeView]
115
- # @return [void]
116
- def probe_lines(view)
117
- if view.final && view.final != view.requested
118
- puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
112
+ private
113
+
114
+ ##
115
+ # @param view [ProbeView]
116
+ # @return [void]
117
+ def probe_lines(view)
118
+ if view.final && view.final != view.requested
119
+ puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
120
+ end
121
+ puts " Surface: #{view.surface} (#{view.articles_count} articles)"
119
122
  end
120
- puts " Surface: #{view.surface} (#{view.articles_count} articles)"
121
123
  end
122
- private_class_method :probe_lines
123
124
  end
124
125
  end
125
126
  end
@@ -5,15 +5,13 @@ module Html2rss
5
5
  ##
6
6
  # Validate command: file/glob/stdin branching and multi-file reporting.
7
7
  module Validate
8
- module_function
9
-
10
8
  ##
11
9
  # @param files [Array<String>]
12
10
  # @param params [Hash]
13
11
  # @param quiet [Boolean]
14
12
  # @return [void]
15
13
  # @raise [Thor::Error]
16
- def run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
14
+ def self.run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
17
15
  if files.size == 2 && File.file?(files[0].to_s) && !File.exist?(files[1].to_s)
18
16
  raise Thor::Error, "No such file: #{files[1]}" if path_like_config?(files[1])
19
17
 
@@ -45,7 +43,7 @@ module Html2rss
45
43
  ##
46
44
  # @param files [Array<String>]
47
45
  # @return [Array<String>]
48
- def resolve_files(files)
46
+ def self.resolve_files(files)
49
47
  return ['-'] if files.empty? || files == ['-']
50
48
 
51
49
  resolved = []
data/lib/html2rss/cli.rb CHANGED
@@ -114,24 +114,26 @@ module Html2rss
114
114
  method_option :explain, type: :boolean, desc: 'Print capture diagnostics to stderr', default: false
115
115
  # @param target [String, nil] URL or local HTML file
116
116
  # @return [void]
117
- def capture(target = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
117
+ def capture(target = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
118
118
  strategy, local_file_path, url = prepare_auto_inputs(target, options[:input])
119
119
  topics = options[:topics]&.split(',')&.map(&:strip)
120
120
 
121
- result = Html2rss::Capture.build(
122
- url,
123
- strategy:,
124
- items_selector: options[:items_selector],
125
- topics:,
126
- title: options[:title],
127
- summary: options[:summary],
128
- force: options[:force],
129
- enhance: options[:enhance],
130
- limit: options[:limit]&.to_i,
131
- max_redirects: options[:max_redirects],
132
- max_requests: options[:max_requests],
133
- local_file_path:
134
- )
121
+ result = execute_feed do
122
+ Html2rss::Capture.build(
123
+ url,
124
+ strategy:,
125
+ items_selector: options[:items_selector],
126
+ topics:,
127
+ title: options[:title],
128
+ summary: options[:summary],
129
+ force: options[:force],
130
+ enhance: options[:enhance],
131
+ limit: options[:limit],
132
+ max_redirects: options[:max_redirects],
133
+ max_requests: options[:max_requests],
134
+ local_file_path:
135
+ )
136
+ end
135
137
 
136
138
  if result.native_feed? && !options[:force]
137
139
  $stderr.puts "First-party RSS/Atom feed detected at #{result.native_feed}. Use --force to capture anyway." # rubocop:disable Style/StderrPuts
@@ -158,15 +160,17 @@ module Html2rss
158
160
  # @return [void]
159
161
  def test(config_input = nil, feed_name = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
160
162
  raw_content, input_source = read_config_input(config_input)
161
- result = Html2rss.test(
162
- raw_content,
163
- feed_name,
164
- min_items: options.fetch(:min_items, 1).to_i,
165
- params: options[:params] || {},
166
- strategy: options[:strategy],
167
- strict_quality: options.fetch(:strict_quality, false),
168
- compare_enhance: options.fetch(:compare_enhance, false)
169
- )
163
+ result = execute_feed do
164
+ Html2rss.test(
165
+ raw_content,
166
+ feed_name,
167
+ min_items: options.fetch(:min_items, 1),
168
+ params: options[:params] || {},
169
+ strategy: options[:strategy],
170
+ strict_quality: options.fetch(:strict_quality, false),
171
+ compare_enhance: options.fetch(:compare_enhance, false)
172
+ )
173
+ end
170
174
 
171
175
  if options[:json]
172
176
  puts JSON.pretty_generate(result.to_h)
@@ -202,7 +206,7 @@ module Html2rss
202
206
  Config.from_yaml(raw_content)
203
207
  end
204
208
  config[:params] = options[:params] || {}
205
- apply_runtime_request_overrides!(config)
209
+ execute_feed { apply_runtime_request_overrides!(config) }
206
210
  apply_local_file_input!(config, options[:input]) if options[:input]
207
211
 
208
212
  run_feed_command { Html2rss.apply(config) }
@@ -421,7 +425,9 @@ module Html2rss
421
425
  Html2rss::RequestService::BotasaurusConnectionFailed,
422
426
  Html2rss::RequestService::BotasaurusServiceError,
423
427
  Html2rss::RequestService::BlockedSurfaceDetected,
424
- Html2rss::NoFeedItemsExtracted => error
428
+ Html2rss::NoFeedItemsExtracted,
429
+ Html2rss::InvalidOption,
430
+ Html2rss::Url::InvalidUrlError => error
425
431
  raise Thor::Error, error.message
426
432
  end
427
433
 
@@ -433,7 +439,7 @@ module Html2rss
433
439
  max_redirects: options[:max_redirects],
434
440
  max_requests: options[:max_requests],
435
441
  local_file_path:,
436
- limit: options[:limit]&.to_i,
442
+ limit: options[:limit],
437
443
  keep_different_domain: options[:keep_different_domain]
438
444
  )
439
445
  end
@@ -52,6 +52,8 @@ module Html2rss
52
52
  # @param max_requests [Integer, nil]
53
53
  # @return [RequestControls]
54
54
  def self.from_shortcut(strategy:, max_redirects: nil, max_requests: nil)
55
+ strategy = Option.strategy(strategy) unless strategy.nil?
56
+ check_budgets!(max_redirects, max_requests)
55
57
  keys = []
56
58
  keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
57
59
  keys << :max_redirects unless max_redirects.nil?
@@ -68,7 +70,8 @@ module Html2rss
68
70
  # @param max_requests [Integer, nil]
69
71
  # @return [RequestControls]
70
72
  def self.from_cli_options(strategy:, max_redirects: nil, max_requests: nil)
71
- strategy_sym = strategy&.to_sym
73
+ strategy_sym = Option.strategy(strategy) if strategy
74
+ check_budgets!(max_redirects, max_requests)
72
75
  keys = []
73
76
  keys << :strategy if strategy
74
77
  keys << :max_redirects unless max_redirects.nil?
@@ -76,7 +79,12 @@ module Html2rss
76
79
  new(strategy: strategy_sym, max_redirects:, max_requests:, explicit_keys: keys)
77
80
  end
78
81
 
79
- private_class_method :explicit_keys_for, :request_value_for, :request_key?
82
+ def self.check_budgets!(max_redirects, max_requests)
83
+ Option.integer(:max_redirects, max_redirects, min: 0, optional: true)
84
+ Option.integer(:max_requests, max_requests, min: 1, optional: true)
85
+ end
86
+
87
+ private_class_method :explicit_keys_for, :request_value_for, :request_key?, :check_budgets!
80
88
 
81
89
  ##
82
90
  # @param strategy [Symbol, nil] effective request strategy
@@ -7,8 +7,6 @@ module Html2rss
7
7
  ##
8
8
  # JSON export adapter from the runtime validators to the packaged schema artifact.
9
9
  module Schema
10
- module_function
11
-
12
10
  # Canonical filename for the exported config JSON schema artifact.
13
11
  SCHEMA_FILENAME = 'html2rss-config.schema.json'
14
12
 
@@ -16,7 +14,7 @@ module Html2rss
16
14
  # Returns the exported configuration JSON Schema.
17
15
  #
18
16
  # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
19
- def json_schema
17
+ def self.json_schema
20
18
  load_json_schema_extension!
21
19
  Builder.call
22
20
  end
@@ -26,7 +24,7 @@ module Html2rss
26
24
  #
27
25
  # @param pretty [Boolean] whether to pretty-print the JSON output
28
26
  # @return [String] serialized JSON Schema
29
- def json_schema_json(pretty: true)
27
+ def self.json_schema_json(pretty: true)
30
28
  pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
31
29
  end
32
30
 
@@ -34,7 +32,7 @@ module Html2rss
34
32
  # Resolves the packaged schema path used by downstream tools.
35
33
  #
36
34
  # @return [String] absolute path to the packaged JSON schema file
37
- def path
35
+ def self.path
38
36
  search_path = File.expand_path(__dir__)
39
37
 
40
38
  loop do
@@ -51,7 +49,7 @@ module Html2rss
51
49
  end
52
50
 
53
51
  # @return [void]
54
- def load_json_schema_extension!
52
+ def self.load_json_schema_extension!
55
53
  require 'dry/schema/extensions/json_schema'
56
54
  Dry::Schema.load_extensions(:json_schema)
57
55
  end
@@ -221,7 +219,7 @@ module Html2rss
221
219
  #
222
220
  # @param schema [Hash] dry-schema JSON schema fragment for a selector
223
221
  # @return [Hash] same schema with registry $ref wiring applied
224
- def apply_selector_registry_refs!(schema)
222
+ def self.apply_selector_registry_refs!(schema)
225
223
  properties = schema.fetch(:properties)
226
224
  properties[:extractor] = extractor_wire_schema
227
225
  properties[:post_process] = post_process_wire_schema
@@ -229,7 +227,7 @@ module Html2rss
229
227
  end
230
228
 
231
229
  # @return [Hash{Symbol => Object}]
232
- def extractor_wire_schema
230
+ def self.extractor_wire_schema
233
231
  {
234
232
  description: 'Extractor used to pull a value from the selected element.',
235
233
  oneOf: registry_ref_list('extractors', Selectors::Extractors::NAME_TO_CLASS)
@@ -237,7 +235,7 @@ module Html2rss
237
235
  end
238
236
 
239
237
  # @return [Hash{Symbol => Object}]
240
- def post_process_wire_schema
238
+ def self.post_process_wire_schema
241
239
  {
242
240
  type: 'array',
243
241
  description: 'Ordered transforms applied to the extracted value.',
@@ -251,7 +249,7 @@ module Html2rss
251
249
  # @param catalog [String] $defs catalog name (`extractors` or `post_processors`)
252
250
  # @param registry [Hash{Symbol => Class}]
253
251
  # @return [Array<Hash>] oneOf entries of `{ '$ref' => ... }`
254
- def registry_ref_list(catalog, registry)
252
+ def self.registry_ref_list(catalog, registry)
255
253
  registry.keys.sort.map do |name|
256
254
  { '$ref' => "#/$defs/#{catalog}/#{name}" }
257
255
  end
@@ -216,6 +216,7 @@ module Html2rss
216
216
  def array_selector_shape_error(key, selector)
217
217
  return "`#{key}` must be an array" unless selector.is_a?(Array)
218
218
  return "`#{key}` must contain at least one element" if selector.empty?
219
+ return "`#{key}` must contain only selector names" unless selector.all? { _1 in String | Symbol }
219
220
 
220
221
  nil
221
222
  end
@@ -158,15 +158,15 @@ module Html2rss
158
158
  if (url_string = values.dig(:channel, :url)) && !url_string.empty?
159
159
  begin
160
160
  Html2rss::Url.for_channel(url_string)
161
- rescue ArgumentError => error
161
+ rescue Html2rss::Url::InvalidUrlError => error
162
162
  key(%i[channel url]).failure(error.message)
163
163
  end
164
164
  end
165
165
 
166
166
  if (image_string = values.dig(:channel, :image)) && !image_string.empty?
167
167
  begin
168
- Html2rss::Url.from_absolute(image_string)
169
- rescue ArgumentError => error
168
+ Html2rss::Url.from_absolute_web(image_string)
169
+ rescue Html2rss::Url::InvalidUrlError => error
170
170
  key(%i[channel image]).failure(error.message)
171
171
  end
172
172
  end
@@ -131,7 +131,7 @@ module Html2rss
131
131
  def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil,
132
132
  keep_different_domain: nil)
133
133
  auto_source = AutoSource::DEFAULT_CONFIG
134
- auto_source = auto_source.merge(limit:) unless limit.nil?
134
+ auto_source = auto_source.merge(limit: Option.integer(:limit, limit, min: 1)) unless limit.nil?
135
135
  unless keep_different_domain.nil?
136
136
  auto_source = auto_source.merge(
137
137
  cleanup: auto_source[:cleanup].merge(keep_different_domain:)
@@ -57,7 +57,7 @@ module Html2rss
57
57
  # @return [Logger] the logger
58
58
  # @raise [ArgumentError] if logger is not a Logger
59
59
  def logger=(logger)
60
- raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(::Logger)
60
+ raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(Logger)
61
61
 
62
62
  @logger = logger
63
63
  @logger.level = @log_level
@@ -114,20 +114,9 @@ module Html2rss
114
114
  #
115
115
  # @param strategy [Symbol, String, nil] the strategy plan name
116
116
  # @return [Symbol, nil] the normalized strategy plan name
117
- # @raise [ArgumentError] if the strategy plan is unknown
117
+ # @raise [InvalidOption] if the strategy plan is unknown
118
118
  def default_strategy=(strategy)
119
- if strategy.nil?
120
- @default_strategy = nil
121
- else
122
- unless strategy.is_a?(Symbol) || strategy.is_a?(String)
123
- raise ArgumentError, 'strategy must be a Symbol or String'
124
- end
125
-
126
- normalized = strategy.to_sym
127
- raise ArgumentError, "unknown strategy: #{strategy}" unless FeedPipeline::StrategyPlan.valid?(normalized)
128
-
129
- @default_strategy = normalized
130
- end
119
+ @default_strategy = (Option.strategy(strategy) unless strategy.nil?)
131
120
  end
132
121
 
133
122
  ##
@@ -171,8 +160,8 @@ module Html2rss
171
160
  # @return [void]
172
161
  def initialize_copy(other)
173
162
  super
174
- @headers = @headers.dup if @headers.is_a?(Hash)
175
- @stylesheets = @stylesheets.map(&:dup) if @stylesheets.is_a?(Array)
163
+ @headers = other.headers.dup if other.headers.is_a?(Hash)
164
+ @stylesheets = other.stylesheets.map(&:dup)
176
165
  end
177
166
 
178
167
  private
@@ -32,15 +32,13 @@ module Html2rss
32
32
  end
33
33
  end
34
34
 
35
- module_function
36
-
37
35
  ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
38
36
  private_constant :ENV_VAR
39
37
 
40
38
  ##
41
39
  # @param sample_url [String, nil] optional URL for a sample scrape smoke test
42
40
  # @return [Result]
43
- def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
41
+ def self.call(sample_url: nil) # rubocop:disable Metrics/AbcSize
44
42
  env = env_check
45
43
  checks = [env_check_record(env)]
46
44
  return missing_env_failure(checks) unless checks.first.ok
@@ -57,7 +55,7 @@ module Html2rss
57
55
 
58
56
  ##
59
57
  # @return [Hash{Symbol => Object}]
60
- def env_check
58
+ def self.env_check
61
59
  base = ENV.fetch(ENV_VAR, '').strip
62
60
  {
63
61
  configured: MCP::Runtime.botasaurus_configured?,
@@ -69,7 +67,7 @@ module Html2rss
69
67
  ##
70
68
  # @param base_url [String]
71
69
  # @return [Hash{Symbol => Object}]
72
- def health_check(base_url)
70
+ def self.health_check(base_url)
73
71
  endpoint = "#{Url.for_channel(base_url).to_s.chomp('/')}/health"
74
72
  response = health_session.get(endpoint)
75
73
  raise response.error if response.is_a?(HTTPX::ErrorResponse)
@@ -80,60 +78,54 @@ module Html2rss
80
78
  { ok: false, error: "#{error.class}: #{error.message}" }
81
79
  end
82
80
 
83
- def health_session
81
+ def self.health_session
84
82
  HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
85
83
  resolver_class: :system)
86
84
  end
87
- module_function :health_session
88
85
  private_class_method :health_session
89
86
 
90
87
  ##
91
88
  # @param url [String]
92
89
  # @return [Hash{Symbol => Object}]
93
- def sample_scrape(url)
90
+ def self.sample_scrape(url)
94
91
  wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
95
92
  {
96
93
  ok: wire[:status].to_i.between?(200, 399),
97
94
  status: wire[:status],
98
95
  articles_count: wire[:articles_count]
99
96
  }
100
- rescue StandardError => error
97
+ rescue Html2rss::Error => error
101
98
  { ok: false, error: "#{error.class}: #{error.message}" }
102
99
  end
103
100
 
104
- def env_check_record(env = env_check)
101
+ def self.env_check_record(env = env_check)
105
102
  Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
106
103
  end
107
- module_function :env_check_record
108
104
  private_class_method :env_check_record
109
105
 
110
- def health_check_record(base_url)
106
+ def self.health_check_record(base_url)
111
107
  return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
112
108
 
113
109
  health = health_check(base_url)
114
110
  Check.new(name: :health, ok: health[:ok], detail: health)
115
111
  end
116
- module_function :health_check_record
117
112
  private_class_method :health_check_record
118
113
 
119
- def sample_scrape_record(url)
114
+ def self.sample_scrape_record(url)
120
115
  sample = sample_scrape(url)
121
116
  Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
122
117
  end
123
- module_function :sample_scrape_record
124
118
  private_class_method :sample_scrape_record
125
119
 
126
- def missing_env_failure(checks)
120
+ def self.missing_env_failure(checks)
127
121
  Log.info("doctor botasaurus: #{ENV_VAR} unset")
128
122
  failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
129
123
  end
130
- module_function :missing_env_failure
131
124
  private_class_method :missing_env_failure
132
125
 
133
- def failure(checks, message)
126
+ def self.failure(checks, message)
134
127
  Result.new(ok: false, checks:, message:)
135
128
  end
136
- module_function :failure
137
129
  private_class_method :failure
138
130
  end
139
131
  end