html2rss 0.32.4 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. data/lib/html2rss/article/enclosure.rb +2 -6
  3. data/lib/html2rss/auto_source/cleanup.rb +1 -3
  4. data/lib/html2rss/auto_source/scraper/html.rb +2 -8
  5. data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
  6. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
  7. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
  8. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  9. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
  10. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
  11. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
  12. data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
  13. data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
  14. data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
  15. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  16. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
  17. data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
  18. data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
  19. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
  20. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  21. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
  22. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  23. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
  25. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
  26. data/lib/html2rss/auto_source/scraper.rb +0 -11
  27. data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
  28. data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
  29. data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
  30. data/lib/html2rss/auto_source.rb +1 -1
  31. data/lib/html2rss/batch.rb +22 -32
  32. data/lib/html2rss/capture/README.md +1 -1
  33. data/lib/html2rss/capture.rb +7 -27
  34. data/lib/html2rss/cli/render.rb +97 -96
  35. data/lib/html2rss/cli/validate.rb +2 -4
  36. data/lib/html2rss/cli.rb +33 -27
  37. data/lib/html2rss/config/request_controls.rb +10 -2
  38. data/lib/html2rss/config/schema.rb +8 -10
  39. data/lib/html2rss/config/selectors_validator.rb +1 -0
  40. data/lib/html2rss/config/validator.rb +3 -3
  41. data/lib/html2rss/config.rb +1 -1
  42. data/lib/html2rss/defaults.rb +5 -16
  43. data/lib/html2rss/doctor/botasaurus.rb +11 -19
  44. data/lib/html2rss/error.rb +7 -3
  45. data/lib/html2rss/feed_builder/rss.rb +1 -1
  46. data/lib/html2rss/feed_pipeline/auto_fallback.rb +3 -3
  47. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  48. data/lib/html2rss/feed_resolution/options.rb +3 -7
  49. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  50. data/lib/html2rss/feed_resolution/probe.rb +3 -3
  51. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  52. data/lib/html2rss/feed_resolution.rb +5 -9
  53. data/lib/html2rss/feed_result.rb +4 -3
  54. data/lib/html2rss/hash_util.rb +91 -90
  55. data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
  56. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  57. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
  58. data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
  59. data/lib/html2rss/html/probe.rb +51 -51
  60. data/lib/html2rss/link_destination/path_classifier.rb +11 -33
  61. data/lib/html2rss/mcp/config_argument.rb +9 -2
  62. data/lib/html2rss/mcp/contract.rb +7 -5
  63. data/lib/html2rss/mcp/outcome.rb +4 -5
  64. data/lib/html2rss/mcp/runtime.rb +17 -24
  65. data/lib/html2rss/mcp/server/tools.rb +9 -9
  66. data/lib/html2rss/option.rb +48 -0
  67. data/lib/html2rss/page_recon/diagnostics.rb +7 -9
  68. data/lib/html2rss/page_recon.rb +0 -4
  69. data/lib/html2rss/recon.rb +10 -12
  70. data/lib/html2rss/request_service/blocked_surface.rb +3 -7
  71. data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
  72. data/lib/html2rss/request_service/budget.rb +3 -24
  73. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  74. data/lib/html2rss/request_service/context.rb +1 -3
  75. data/lib/html2rss/request_service/local_file_strategy.rb +0 -1
  76. data/lib/html2rss/request_service/network_guard.rb +3 -2
  77. data/lib/html2rss/request_service/policy.rb +8 -26
  78. data/lib/html2rss/request_service/response.rb +3 -6
  79. data/lib/html2rss/request_session/pager/base.rb +3 -3
  80. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  81. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  82. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  83. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  84. data/lib/html2rss/scoring/engine.rb +4 -32
  85. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  86. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  87. data/lib/html2rss/selectors/json_xml.rb +33 -34
  88. data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
  89. data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
  90. data/lib/html2rss/selectors/schema_export.rb +60 -60
  91. data/lib/html2rss/selectors.rb +14 -18
  92. data/lib/html2rss/sst/node.rb +9 -3
  93. data/lib/html2rss/sst/normalizer.rb +25 -24
  94. data/lib/html2rss/sst/text.rb +53 -57
  95. data/lib/html2rss/status.rb +23 -30
  96. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  97. data/lib/html2rss/syndication/discovery.rb +117 -187
  98. data/lib/html2rss/syndication/parser.rb +92 -115
  99. data/lib/html2rss/test/enhance_audit.rb +13 -15
  100. data/lib/html2rss/test/policy.rb +9 -11
  101. data/lib/html2rss/test.rb +29 -48
  102. data/lib/html2rss/url.rb +53 -53
  103. data/lib/html2rss/version.rb +1 -1
  104. data/lib/html2rss.rb +2 -1
  105. metadata +4 -2
@@ -6,136 +6,113 @@ module Html2rss
6
6
  module Syndication
7
7
  ##
8
8
  # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
9
- module Parser # rubocop:disable Metrics/ModuleLength -- feed formats + structured parse warn
10
- module_function
11
-
12
- ##
13
- # @param response [Html2rss::RequestService::Response]
14
- # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
15
- # @raise [ArgumentError] when the response is not a syndication feed
16
- def parse_response(response)
17
- raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
9
+ module Parser
10
+ class << self
11
+ ##
12
+ # @param response [Html2rss::RequestService::Response]
13
+ # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
14
+ # @raise [ArgumentError] when the response is not a syndication feed
15
+ def parse_response(response)
16
+ raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
17
+
18
+ parse(response.body, base_url: response.url)
19
+ end
18
20
 
19
- parse(response.body, base_url: response.url)
20
- end
21
+ ##
22
+ # @param body [String] raw RSS/Atom document
23
+ # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
24
+ # @return [Array<Hash{Symbol => Object}>]
25
+ def parse(body, base_url: nil)
26
+ feed = RSS::Parser.parse(body.to_s, false)
27
+
28
+ items_from(feed).filter_map { |item| article_hash(item, base_url:) }
29
+ rescue RSS::Error => error
30
+ Log.warn({
31
+ event_name: 'syndication.parse_failed',
32
+ error_class: error.class.name,
33
+ message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
34
+ }.to_json)
35
+ []
36
+ end
21
37
 
22
- ##
23
- # @param body [String] raw RSS/Atom document
24
- # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
25
- # @return [Array<Hash{Symbol => Object}>]
26
- def parse(body, base_url: nil)
27
- feed = RSS::Parser.parse(body.to_s, false)
28
- return [] unless feed
29
-
30
- items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
31
- rescue RSS::Error => error
32
- Log.warn({
33
- event_name: 'syndication.parse_failed',
34
- error_class: error.class.name,
35
- message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
36
- }.to_json)
37
- []
38
- end
38
+ private
39
39
 
40
- def items_from(feed, base_url:) # rubocop:disable Lint/UnusedMethodArgument -- base reserved for callers
41
- case feed
42
- when RSS::Rss then Array(feed.items)
43
- when RSS::Atom::Feed then Array(feed.entries)
44
- else []
40
+ def items_from(feed)
41
+ case feed
42
+ when RSS::Rss then feed.items
43
+ when RSS::Atom::Feed then feed.entries
44
+ else []
45
+ end
45
46
  end
46
- end
47
- module_function :items_from
48
- private_class_method :items_from
49
47
 
50
- def article_hash(item, base_url:)
51
- case item
52
- when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
53
- when RSS::Atom::Entry, RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
48
+ def article_hash(item, base_url:)
49
+ case item
50
+ when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
51
+ when RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
52
+ end
54
53
  end
55
- end
56
- module_function :article_hash
57
- private_class_method :article_hash
58
-
59
- def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
60
- url = absolute_url(item.link, base_url:)
61
- return unless url
62
-
63
- {
64
- id: present(item.guid&.content) || url.to_s,
65
- title: present(item.title),
66
- description: present(item.description),
67
- url:,
68
- published_at: item.date || item.pubDate,
69
- author: present(item.author),
70
- categories: Array(item.categories).filter_map { |cat| present(cat.content) }
71
- }.compact
72
- end
73
- module_function :rss_item_hash
74
- private_class_method :rss_item_hash
75
-
76
- def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
77
- url = atom_link(entry, base_url:)
78
- return unless url
79
-
80
- {
81
- id: present(entry.id&.content) || url.to_s,
82
- title: present(atom_text(entry.title)),
83
- description: present(atom_text(entry.content) || atom_text(entry.summary)),
84
- url:,
85
- published_at: entry.published&.content || entry.updated&.content,
86
- author: present(atom_author(entry)),
87
- categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
88
- }.compact
89
- end
90
- module_function :atom_entry_hash
91
- private_class_method :atom_entry_hash
92
54
 
93
- def atom_link(entry, base_url:)
94
- links = Array(entry.links)
95
- link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
96
- absolute_url(link&.href, base_url:)
97
- end
98
- module_function :atom_link
99
- private_class_method :atom_link
55
+ def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
56
+ url = absolute_url(item.link, base_url:)
57
+ return unless url
58
+
59
+ {
60
+ id: present(item.guid&.content) || url.to_s,
61
+ title: present(item.title),
62
+ description: present(item.description),
63
+ url:,
64
+ published_at: item.pubDate,
65
+ author: present(item.author),
66
+ categories: item.categories.filter_map { |cat| present(cat.content) }
67
+ }.compact
68
+ end
100
69
 
101
- def atom_text(node)
102
- return unless node.is_a?(RSS::Atom::TextConstruct) || node.is_a?(RSS::Atom::Feed::Entry::Content)
70
+ def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
71
+ url = atom_link(entry, base_url:)
72
+ return unless url
73
+
74
+ {
75
+ id: present(entry.id&.content) || url.to_s,
76
+ title: present(entry.title&.content),
77
+ description: present(entry.content&.content || entry.summary&.content),
78
+ url:,
79
+ published_at: entry.published&.content || entry.updated&.content,
80
+ author: atom_author(entry),
81
+ categories: entry.categories.filter_map { |cat| present(cat.term) }
82
+ }.compact
83
+ end
103
84
 
104
- node.content
105
- end
106
- module_function :atom_text
107
- private_class_method :atom_text
85
+ def atom_link(entry, base_url:)
86
+ links = entry.links
87
+ link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
88
+ absolute_url(link&.href, base_url:)
89
+ end
108
90
 
109
- def atom_author(entry)
110
- author = Array(entry.author).first
111
- return unless author
91
+ def atom_author(entry)
92
+ author = entry.author
93
+ return unless author
112
94
 
113
- present(author.name&.content) || present(author.email&.content)
114
- end
115
- module_function :atom_author
116
- private_class_method :atom_author
117
-
118
- def absolute_url(raw, base_url:)
119
- value = present(raw)
120
- return unless value
95
+ present(author.name&.content) || present(author.email&.content)
96
+ end
121
97
 
122
- if base_url
123
- Html2rss::Url.from_relative(value, base_url)
124
- else
125
- Html2rss::Url.from_absolute(value)
98
+ def absolute_url(raw, base_url:)
99
+ value = present(raw)
100
+ return unless value
101
+
102
+ if base_url
103
+ Html2rss::Url.from_relative(value, base_url)
104
+ else
105
+ Html2rss::Url.from_absolute(value)
106
+ end
107
+ rescue Html2rss::Url::InvalidUrlError
108
+ nil
126
109
  end
127
- rescue ArgumentError
128
- nil
129
- end
130
- module_function :absolute_url
131
- private_class_method :absolute_url
132
110
 
133
- def present(value)
134
- text = value.to_s.strip
135
- text unless text.empty?
111
+ def present(value)
112
+ text = value.to_s.strip
113
+ text unless text.empty?
114
+ end
136
115
  end
137
- module_function :present
138
- private_class_method :present
139
116
  end
140
117
  end
141
118
  end
@@ -6,8 +6,6 @@ module Html2rss
6
6
  # Probes list-card +enhance+ value on cached listing HTML.
7
7
  # Sole owner of enhance_gains metrics and enhance-specific warn-only warnings.
8
8
  module EnhanceAudit # rubocop:disable Metrics/ModuleLength -- probe + compare + heuristics stay co-located
9
- module_function
10
-
11
9
  TRACKED_KEYS = %i[title url description published_at categories author image id enclosures].freeze
12
10
  CURATOR_KEYS = %i[title url description published_at].freeze
13
11
  private_constant :TRACKED_KEYS, :CURATOR_KEYS
@@ -28,7 +26,7 @@ module Html2rss
28
26
  # @param selectors [Hash{Symbol => Object}]
29
27
  # @param time_zone [String]
30
28
  # @return [AuditSlice, nil] nil when selectors are absent
31
- def probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
29
+ def self.probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
32
30
  return nil unless selectors&.dig(:items, :selector)
33
31
 
34
32
  selector_engine = Selectors.new(response, selectors:, time_zone:)
@@ -64,7 +62,7 @@ module Html2rss
64
62
  # @param selectors [Hash{Symbol => Object}]
65
63
  # @param time_zone [String]
66
64
  # @return [Hash{Symbol => Object}, nil]
67
- def compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
65
+ def self.compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
68
66
  return nil unless selectors&.dig(:items, :selector)
69
67
 
70
68
  off = extraction_stats(response, selectors, time_zone, enhance: false)
@@ -84,7 +82,7 @@ module Html2rss
84
82
 
85
83
  ##
86
84
  # @return [EnhanceGains] zeroed gains for compare fallback
87
- def empty_gains
85
+ def self.empty_gains
88
86
  EnhanceGains.new(
89
87
  items_probed: 0,
90
88
  keys_added: zero_keys_added,
@@ -93,12 +91,12 @@ module Html2rss
93
91
  )
94
92
  end
95
93
 
96
- def zero_keys_added
94
+ def self.zero_keys_added
97
95
  TRACKED_KEYS.to_h { |key| [key, 0] }.freeze
98
96
  end
99
97
  private_class_method :zero_keys_added
100
98
 
101
- def extraction_stats(response, selectors, time_zone, enhance:)
99
+ def self.extraction_stats(response, selectors, time_zone, enhance:)
102
100
  merged = selectors_with_enhance(selectors, enhance:)
103
101
  articles = Selectors.new(response, selectors: merged, time_zone:).articles
104
102
  {
@@ -108,14 +106,14 @@ module Html2rss
108
106
  end
109
107
  private_class_method :extraction_stats
110
108
 
111
- def selectors_with_enhance(selectors, enhance:)
109
+ def self.selectors_with_enhance(selectors, enhance:)
112
110
  merged = HashUtil.deep_dup(selectors)
113
111
  merged[:items] = merged.fetch(:items, {}).merge(enhance:)
114
112
  merged
115
113
  end
116
114
  private_class_method :selectors_with_enhance
117
115
 
118
- def keys_added(baseline, enhanced)
116
+ def self.keys_added(baseline, enhanced)
119
117
  enhanced.each_with_object({}) do |(key, value), counts|
120
118
  next if value.nil?
121
119
 
@@ -127,7 +125,7 @@ module Html2rss
127
125
  end
128
126
  private_class_method :keys_added
129
127
 
130
- def merge_keys_added!(gains, item_keys)
128
+ def self.merge_keys_added!(gains, item_keys)
131
129
  gains[:items_probed] += 1
132
130
  item_keys.each do |key, count|
133
131
  next unless TRACKED_KEYS.include?(key)
@@ -138,13 +136,13 @@ module Html2rss
138
136
  end
139
137
  private_class_method :merge_keys_added!
140
138
 
141
- def finalize_gains!(gains)
139
+ def self.finalize_gains!(gains)
142
140
  curator_total = CURATOR_KEYS.sum { |key| gains[:keys_added][key] }
143
141
  gains[:no_op] = curator_total.zero?
144
142
  end
145
143
  private_class_method :finalize_gains!
146
144
 
147
- def append_item_warnings!(warnings, item:, enhanced:, item_keys:)
145
+ def self.append_item_warnings!(warnings, item:, enhanced:, item_keys:)
148
146
  curator_added = item_keys.slice(*CURATOR_KEYS)
149
147
  return if curator_added.empty?
150
148
  return unless curator_added.key?(:description)
@@ -160,7 +158,7 @@ module Html2rss
160
158
  end
161
159
  private_class_method :append_item_warnings!
162
160
 
163
- def category_only_description?(description, enhanced)
161
+ def self.category_only_description?(description, enhanced)
164
162
  normalized = description.downcase
165
163
  return true if Html2rss::Html::ArticleRules::Description::TYPE_CHIPS.include?(normalized)
166
164
 
@@ -169,7 +167,7 @@ module Html2rss
169
167
  end
170
168
  private_class_method :category_only_description?
171
169
 
172
- def image_only_description?(description, item)
170
+ def self.image_only_description?(description, item)
173
171
  images = item.css('img')
174
172
  return description == images.first['alt'].to_s.strip if images.size == 1 && images.first['alt']
175
173
 
@@ -177,7 +175,7 @@ module Html2rss
177
175
  end
178
176
  private_class_method :image_only_description?
179
177
 
180
- def prose_shaped?(text)
178
+ def self.prose_shaped?(text)
181
179
  return false if text.match?(IMAGE_FILENAME)
182
180
  return false if text.match?(%r{\Ahttps?://}i)
183
181
  return false if text.match?(%r{\A/[\w./-]+\z})
@@ -6,8 +6,6 @@ module Html2rss
6
6
  # Pass/fail classification for a configuration test after live extraction.
7
7
  # Owns min_items vs quality thresholds; {Test.call} orchestrates only.
8
8
  module Policy
9
- module_function
10
-
11
9
  Outcome = Data.define(:passed, :failure_kind, :error_message)
12
10
 
13
11
  ##
@@ -28,7 +26,7 @@ module Html2rss
28
26
  # @param strict_quality [Boolean]
29
27
  # @param quality_report [QualityReport]
30
28
  # @return [Outcome]
31
- def evaluate(item_count:, min_items:, strict_quality:, quality_report:)
29
+ def self.evaluate(item_count:, min_items:, strict_quality:, quality_report:)
32
30
  min_items_passed = item_count >= min_items
33
31
  quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
34
32
  passed = min_items_passed && !quality_failed
@@ -39,7 +37,7 @@ module Html2rss
39
37
  Outcome.new(passed:, failure_kind:, error_message:)
40
38
  end
41
39
 
42
- def log_strict_quality_failure(failure_kind, item_count)
40
+ def self.log_strict_quality_failure(failure_kind, item_count)
43
41
  Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
44
42
  end
45
43
  private_class_method :log_strict_quality_failure
@@ -47,7 +45,7 @@ module Html2rss
47
45
  ##
48
46
  # @param quality_report [QualityReport]
49
47
  # @return [Boolean]
50
- def quality_failure?(quality_report)
48
+ def self.quality_failure?(quality_report)
51
49
  metrics = quality_report.metrics
52
50
  item_count = metrics[:item_count]
53
51
  return false if item_count.zero?
@@ -62,7 +60,7 @@ module Html2rss
62
60
  # @param min_items [Integer]
63
61
  # @param quality_report [QualityReport]
64
62
  # @return [Array(FailureKind, nil), Array(String, nil)]
65
- def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
63
+ def self.outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
66
64
  return [nil, nil] if min_items_passed && !quality_failed
67
65
  unless min_items_passed
68
66
  return [FailureKind[:min_items],
@@ -72,13 +70,13 @@ module Html2rss
72
70
  [FailureKind[:quality], quality_failure_message(quality_report)]
73
71
  end
74
72
 
75
- def quality_failure_message(quality_report)
73
+ def self.quality_failure_message(quality_report)
76
74
  reasons = quality_failure_reasons(quality_report.metrics)
77
75
  "Feed quality check failed (#{reasons.join(', ')})"
78
76
  end
79
77
  private_class_method :quality_failure_message
80
78
 
81
- def quality_failure_reasons(metrics)
79
+ def self.quality_failure_reasons(metrics)
82
80
  reasons = []
83
81
  reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
84
82
  reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
@@ -87,17 +85,17 @@ module Html2rss
87
85
  end
88
86
  private_class_method :quality_failure_reasons
89
87
 
90
- def duplicate_urls_failure?(metrics)
88
+ def self.duplicate_urls_failure?(metrics)
91
89
  metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
92
90
  end
93
91
  private_class_method :duplicate_urls_failure?
94
92
 
95
- def junk_title_ratio_failure?(metrics)
93
+ def self.junk_title_ratio_failure?(metrics)
96
94
  metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
97
95
  end
98
96
  private_class_method :junk_title_ratio_failure?
99
97
 
100
- def short_title_failure?(metrics)
98
+ def self.short_title_failure?(metrics)
101
99
  metrics[:short_title_count].positive?
102
100
  end
103
101
  private_class_method :short_title_failure?
data/lib/html2rss/test.rb CHANGED
@@ -5,8 +5,6 @@ module Html2rss
5
5
  # Service that runs schema validation and live feed extraction on a configuration,
6
6
  # enforcing minimum item thresholds and capturing sample output.
7
7
  module Test # rubocop:disable Metrics/ModuleLength -- Result + FailureKind nest with owner
8
- module_function
9
-
10
8
  FailureKind = Data.define(:name)
11
9
 
12
10
  ##
@@ -163,27 +161,27 @@ module Html2rss
163
161
  # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
164
162
  # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
165
163
  # @return [Html2rss::Test::Result]
166
- def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
167
- strict_quality: false, compare_enhance: false)
164
+ def self.call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
165
+ strict_quality: false, compare_enhance: false)
168
166
  raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
169
167
  return validation_failure_result(validation, raw_config) unless validation.success?
170
168
 
171
169
  config = build_test_config(raw_config, strategy, params)
172
170
  started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
173
171
  execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
174
- rescue StandardError => error
172
+ rescue Html2rss::Error => error
175
173
  duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
176
174
  execution_failure_result(error, raw_config, duration)
177
175
  end
178
176
 
179
- def build_test_config(raw_config, strategy, params)
177
+ def self.build_test_config(raw_config, strategy, params)
180
178
  raw_config[:strategy] = strategy.to_sym if strategy
181
179
  raw_config[:params] = params if params&.any?
182
180
  Config.from_hash(raw_config)
183
181
  end
184
182
  private_class_method :build_test_config
185
183
 
186
- def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
184
+ def self.compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
187
185
  quality_report_for(
188
186
  feed_result.to_rss.items,
189
187
  channel_url: raw_config.dig(:channel, :url).to_s,
@@ -194,20 +192,20 @@ module Html2rss
194
192
  end
195
193
  private_class_method :compile_quality_report
196
194
 
197
- def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
198
- compare_enhance:)
195
+ def self.execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
196
+ compare_enhance:)
199
197
  feed_result, pipeline_outcome = extract_feed(raw_config)
200
198
  duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
201
199
  quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
202
200
  enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
203
201
 
204
202
  build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
205
- min_items:, strict_quality:, response_body: pipeline_outcome&.response&.body)
203
+ min_items:, strict_quality:, response_body: pipeline_outcome.response.body)
206
204
  end
207
205
  private_class_method :execute_timed_pipeline
208
206
 
209
- def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
210
- min_items:, strict_quality:, response_body:)
207
+ def self.build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
208
+ min_items:, strict_quality:, response_body:)
211
209
  rss_doc = feed_result.to_rss
212
210
  outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
213
211
  strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
@@ -232,27 +230,14 @@ module Html2rss
232
230
  end
233
231
  private_class_method :build_test_result
234
232
 
235
- def extract_samples(items, limit: 3)
233
+ def self.extract_samples(items, limit: 3)
236
234
  items.first(limit).map do |item|
237
- article = feed_item_as_article(item)
238
- {
239
- title: article.title.to_s.strip,
240
- url: article.url.to_s,
241
- published_at: article.published_at
242
- }
235
+ article = Article.new(title: item.title, url: item.link, published_at: item.pubDate)
236
+ { title: article.title.to_s.strip, url: article.url.to_s, published_at: article.published_at }
243
237
  end
244
238
  end
245
239
  private_class_method :extract_samples
246
240
 
247
- def feed_item_as_article(item)
248
- case item
249
- when Article then item
250
- else
251
- Article.new(title: item.title, url: item.link, published_at: item.pubDate)
252
- end
253
- end
254
- private_class_method :feed_item_as_article
255
-
256
241
  ##
257
242
  # Builds a ship-quality report for RSS items (shared by test and MCP apply).
258
243
  #
@@ -263,8 +248,8 @@ module Html2rss
263
248
  # @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
264
249
  # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
265
250
  # @return [QualityReport]
266
- def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
267
- probe_native_feed: true)
251
+ def self.quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
252
+ probe_native_feed: true)
268
253
  audit = AutoSource::Cleanup.audit_feed_items(items)
269
254
  native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
270
255
  report = QualityReport.from_audit(audit, native_feed:)
@@ -272,21 +257,20 @@ module Html2rss
272
257
  merge_enhance_audit(report, config, pipeline_outcome)
273
258
  end
274
259
 
275
- def extract_feed(raw_config)
260
+ def self.extract_feed(raw_config)
276
261
  outcome, feed_result = FeedPipeline.new(raw_config).to_outcome_and_result
277
262
  [feed_result, outcome]
278
263
  end
279
264
  private_class_method :extract_feed
280
265
 
281
- def enhance_enabled?(config)
266
+ def self.enhance_enabled?(config)
282
267
  return false unless config.selectors
283
268
 
284
269
  !!config.selectors.dig(:items, :enhance)
285
270
  end
286
271
  private_class_method :enhance_enabled?
287
272
 
288
- def build_enhance_compare(config, pipeline_outcome)
289
- return nil unless pipeline_outcome
273
+ def self.build_enhance_compare(config, pipeline_outcome)
290
274
  return nil unless config.selectors
291
275
 
292
276
  EnhanceAudit.compare(
@@ -297,9 +281,8 @@ module Html2rss
297
281
  end
298
282
  private_class_method :build_enhance_compare
299
283
 
300
- def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
301
- return report unless pipeline_outcome && enhance_enabled?(config)
302
- return report unless config.selectors
284
+ def self.merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
285
+ return report unless enhance_enabled?(config)
303
286
 
304
287
  slice = EnhanceAudit.probe(
305
288
  response: pipeline_outcome.response,
@@ -320,7 +303,7 @@ module Html2rss
320
303
  end
321
304
  private_class_method :merge_enhance_audit
322
305
 
323
- def enhance_gains_to_h(enhance_gains)
306
+ def self.enhance_gains_to_h(enhance_gains)
324
307
  {
325
308
  items_probed: enhance_gains.items_probed,
326
309
  keys_added: enhance_gains.keys_added,
@@ -330,7 +313,7 @@ module Html2rss
330
313
  end
331
314
  private_class_method :enhance_gains_to_h
332
315
 
333
- def append_url_mismatch_warning(report, channel_url, feed_result)
316
+ def self.append_url_mismatch_warning(report, channel_url, feed_result)
334
317
  return report unless url_mismatch?(channel_url, feed_result)
335
318
 
336
319
  warnings = report.warnings.dup
@@ -344,22 +327,20 @@ module Html2rss
344
327
  end
345
328
  private_class_method :append_url_mismatch_warning
346
329
 
347
- def url_mismatch?(channel_url, feed_result)
348
- status = feed_result&.status
349
- return false unless status
350
-
330
+ def self.url_mismatch?(channel_url, feed_result)
331
+ status = feed_result.status
351
332
  configured = channel_url.to_s
352
333
  final = status.scrape_url.to_s
353
334
  final = status.entry_url.to_s if final.empty?
354
335
  return false if configured.empty? || final.empty?
355
336
 
356
337
  !Url.from_absolute(configured).same_document?(Url.from_absolute(final))
357
- rescue ArgumentError
338
+ rescue Html2rss::Url::InvalidUrlError
358
339
  false
359
340
  end
360
341
  private_class_method :url_mismatch?
361
342
 
362
- def probe_native_feed_url(channel_url, config)
343
+ def self.probe_native_feed_url(channel_url, config)
363
344
  return nil if channel_url.to_s.empty?
364
345
 
365
346
  resources = FeedPipeline::RuntimePolicy.resources_for(config)
@@ -368,12 +349,12 @@ module Html2rss
368
349
  config:, strategy:, budget: resources.budget, policy: resources.policy
369
350
  )
370
351
  Syndication::Discovery.best_feed_url(page_url: channel_url, request_session: session)
371
- rescue StandardError
352
+ rescue Html2rss::Error
372
353
  nil
373
354
  end
374
355
  private_class_method :probe_native_feed_url
375
356
 
376
- def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
357
+ def self.validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
377
358
  Result.new(
378
359
  success: false,
379
360
  item_count: 0,
@@ -391,7 +372,7 @@ module Html2rss
391
372
  end
392
373
  private_class_method :validation_failure_result
393
374
 
394
- def execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
375
+ def self.execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
395
376
  Result.new(
396
377
  success: false,
397
378
  item_count: 0,