html2rss 0.29.1 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -8
  3. data/html2rss.gemspec +1 -3
  4. data/lib/html2rss/article/deduplicator.rb +2 -4
  5. data/lib/html2rss/article.rb +16 -39
  6. data/lib/html2rss/auto_source/README.md +1 -1
  7. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  8. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  10. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  11. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  13. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  14. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  15. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  16. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  17. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  19. data/lib/html2rss/auto_source/scraper.rb +4 -14
  20. data/lib/html2rss/auto_source/segment.rb +7 -4
  21. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  22. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  23. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  24. data/lib/html2rss/auto_source.rb +2 -2
  25. data/lib/html2rss/batch.rb +6 -4
  26. data/lib/html2rss/capture/README.md +7 -5
  27. data/lib/html2rss/capture.rb +68 -36
  28. data/lib/html2rss/channel.rb +3 -3
  29. data/lib/html2rss/cli/probe_view.rb +5 -3
  30. data/lib/html2rss/cli/render.rb +6 -5
  31. data/lib/html2rss/cli/validate.rb +28 -27
  32. data/lib/html2rss/cli.rb +1 -1
  33. data/lib/html2rss/config/issue_mapper.rb +108 -0
  34. data/lib/html2rss/config/request_headers.rb +6 -4
  35. data/lib/html2rss/config/schema.rb +16 -1
  36. data/lib/html2rss/config/selectors_validator.rb +92 -70
  37. data/lib/html2rss/config/validation_issue.rb +60 -0
  38. data/lib/html2rss/config/validation_report.rb +55 -0
  39. data/lib/html2rss/config/validator.rb +10 -6
  40. data/lib/html2rss/config.rb +35 -101
  41. data/lib/html2rss/defaults.rb +42 -30
  42. data/lib/html2rss/doctor/botasaurus.rb +19 -9
  43. data/lib/html2rss/doctor.rb +8 -0
  44. data/lib/html2rss/error.rb +1 -1
  45. data/lib/html2rss/feed_builder/rss.rb +9 -14
  46. data/lib/html2rss/feed_pipeline/README.md +8 -8
  47. data/lib/html2rss/feed_pipeline/auto_fallback.rb +1 -1
  48. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  49. data/lib/html2rss/feed_pipeline/strategy_plan.rb +7 -3
  50. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  51. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  52. data/lib/html2rss/feed_resolution/options.rb +3 -1
  53. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  54. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  55. data/lib/html2rss/html/article_extractor.rb +2 -4
  56. data/lib/html2rss/html/feed_link.rb +3 -1
  57. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  58. data/lib/html2rss/html/navigator.rb +10 -10
  59. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  60. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  61. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  62. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  63. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  64. data/lib/html2rss/mcp/README.md +5 -3
  65. data/lib/html2rss/mcp/config_argument.rb +3 -1
  66. data/lib/html2rss/mcp/contract.rb +15 -25
  67. data/lib/html2rss/mcp/outcome/playbook.rb +7 -7
  68. data/lib/html2rss/mcp/outcome.rb +36 -43
  69. data/lib/html2rss/mcp/server/tools.rb +112 -132
  70. data/lib/html2rss/mcp/server.rb +27 -19
  71. data/lib/html2rss/page_recon/README.md +5 -5
  72. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  73. data/lib/html2rss/page_recon.rb +9 -5
  74. data/lib/html2rss/recon.rb +20 -31
  75. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  76. data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
  77. data/lib/html2rss/request_service/compressed_body.rb +17 -9
  78. data/lib/html2rss/request_service/httpx_strategy.rb +239 -0
  79. data/lib/html2rss/request_service/network_guard.rb +7 -5
  80. data/lib/html2rss/request_service/response.rb +1 -2
  81. data/lib/html2rss/request_service/response_guard.rb +0 -16
  82. data/lib/html2rss/request_service/strategy.rb +22 -11
  83. data/lib/html2rss/request_service.rb +46 -85
  84. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  85. data/lib/html2rss/scoring/engine.rb +8 -8
  86. data/lib/html2rss/scoring/observation.rb +5 -3
  87. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  88. data/lib/html2rss/scrape_target.rb +3 -1
  89. data/lib/html2rss/selector_candidates.rb +234 -0
  90. data/lib/html2rss/selectors/README.md +64 -0
  91. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  92. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  93. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  94. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  95. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  96. data/lib/html2rss/selectors/extractors.rb +33 -20
  97. data/lib/html2rss/selectors/item_env.rb +31 -0
  98. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  99. data/lib/html2rss/selectors/option_spec.rb +140 -0
  100. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  101. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  102. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  103. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  104. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  105. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  106. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  107. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  108. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  109. data/lib/html2rss/selectors/post_processors.rb +6 -5
  110. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  111. data/lib/html2rss/selectors/step_env.rb +24 -0
  112. data/lib/html2rss/selectors.rb +231 -142
  113. data/lib/html2rss/sst/attrs.rb +5 -3
  114. data/lib/html2rss/sst/document.rb +3 -1
  115. data/lib/html2rss/sst/index.rb +3 -1
  116. data/lib/html2rss/sst/node.rb +29 -10
  117. data/lib/html2rss/sst/normalizer.rb +12 -16
  118. data/lib/html2rss/sst/tags.rb +7 -7
  119. data/lib/html2rss/status.rb +13 -8
  120. data/lib/html2rss/surface_category.rb +14 -32
  121. data/lib/html2rss/syndication/parser.rb +2 -2
  122. data/lib/html2rss/test/enhance_audit.rb +1 -3
  123. data/lib/html2rss/test/policy.rb +106 -0
  124. data/lib/html2rss/test.rb +126 -172
  125. data/lib/html2rss/url.rb +25 -4
  126. data/lib/html2rss/version.rb +1 -1
  127. data/lib/html2rss.rb +27 -19
  128. data/lib/tasks/config_schema.rake +2 -2
  129. data/schema/html2rss-config.schema.json +11 -8
  130. metadata +19 -44
  131. data/lib/html2rss/request_service/faraday_strategy.rb +0 -233
  132. data/lib/html2rss/selectors/item_scope.rb +0 -30
data/lib/html2rss/test.rb CHANGED
@@ -12,27 +12,14 @@ module Html2rss
12
12
  ##
13
13
  # Closed failure classification for a failed test run.
14
14
  class FailureKind
15
- # Closed set of test failure wire names.
16
- NAMES = Set[:schema, :execution, :min_items, :quality].freeze
17
-
18
- class << self
19
- ##
20
- # @param value [FailureKind, Symbol, String]
21
- # @return [FailureKind]
22
- def coerce(value)
23
- return value if value.is_a?(self)
24
-
25
- new(name: value.to_sym)
26
- end
27
- end
15
+ # Frozen name → instance table.
16
+ ALL = %i[schema execution min_items quality].to_h { |name| [name, new(name:)] }.freeze
17
+ private_class_method :new
28
18
 
29
19
  ##
30
- # @param name [Symbol]
31
- def initialize(name:)
32
- raise ArgumentError, "unknown failure kind: #{name.inspect}" unless NAMES.include?(name)
33
-
34
- super
35
- end
20
+ # @param name [Symbol, String]
21
+ # @return [FailureKind]
22
+ def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown failure kind: #{name.inspect}" }
36
23
 
37
24
  ##
38
25
  # @return [Boolean]
@@ -59,19 +46,11 @@ module Html2rss
59
46
  def to_s = name.to_s
60
47
  end
61
48
 
49
+ QualityReport = Data.define(:warnings, :metrics, :native_feed, :defer_reason)
50
+
62
51
  ##
63
52
  # Ship-quality audit summary for a configuration test (warn-only).
64
- QualityReport = Data.define(:warnings, :metrics, :native_feed, :defer_reason) do
65
- ##
66
- # @return [Hash{Symbol => Object}]
67
- def to_h
68
- {
69
- warnings: warnings.map(&:to_s),
70
- metrics:,
71
- **(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
72
- }.compact
73
- end
74
-
53
+ class QualityReport
75
54
  ##
76
55
  # @param audit [Html2rss::AutoSource::Cleanup::AuditResult]
77
56
  # @param native_feed [String, nil]
@@ -85,11 +64,18 @@ module Html2rss
85
64
  end
86
65
  new(warnings: report_warnings.freeze, metrics: audit.metrics, native_feed:, defer_reason:)
87
66
  end
67
+
68
+ ##
69
+ # @return [Hash{Symbol => Object}]
70
+ def to_h
71
+ {
72
+ warnings: warnings.map(&:to_s),
73
+ metrics:,
74
+ **(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
75
+ }.compact
76
+ end
88
77
  end
89
78
 
90
- ##
91
- # Immutable outcome of a configuration test. Success carries +rss+ XML from the
92
- # first live extraction; failures carry a typed {FailureKind}.
93
79
  Result = Data.define(
94
80
  :success,
95
81
  :item_count,
@@ -98,13 +84,20 @@ module Html2rss
98
84
  :channel_url,
99
85
  :strategy_used,
100
86
  :duration_seconds,
101
- :validation_errors,
87
+ :validation_issues,
102
88
  :error_message,
103
89
  :failure_kind,
104
90
  :rss,
105
91
  :quality_report,
106
- :enhance_compare
107
- ) do
92
+ :enhance_compare,
93
+ :response_body
94
+ )
95
+
96
+ ##
97
+ # Immutable outcome of a configuration test. Success carries +rss+ XML from the
98
+ # first live extraction; failures carry a typed {FailureKind}.
99
+ # +response_body+ is a copy of the fetched page for in-process readers and stays off {#to_h}.
100
+ class Result
108
101
  ##
109
102
  # @param success [Boolean]
110
103
  # @param item_count [Integer]
@@ -113,22 +106,23 @@ module Html2rss
113
106
  # @param channel_url [String, nil]
114
107
  # @param strategy_used [Symbol, nil]
115
108
  # @param duration_seconds [Float]
116
- # @param validation_errors [Hash, nil]
109
+ # @param validation_issues [Array<Html2rss::Config::ValidationIssue>, nil]
117
110
  # @param error_message [String, nil]
118
111
  # @param failure_kind [FailureKind, nil]
119
112
  # @param rss [String, nil]
120
113
  # @param quality_report [QualityReport, nil]
121
114
  # @param enhance_compare [Hash, nil]
115
+ # @param response_body [String, nil] fetched page HTML copied from the pipeline response
122
116
  def initialize(success:, item_count:, sample_items:, channel_title:, channel_url:, # rubocop:disable Metrics/ParameterLists
123
- strategy_used:, duration_seconds:, validation_errors:, error_message:,
124
- failure_kind:, rss:, quality_report: nil, enhance_compare: nil)
117
+ strategy_used:, duration_seconds:, validation_issues:, error_message:,
118
+ failure_kind:, rss:, quality_report: nil, enhance_compare: nil, response_body: nil)
125
119
  super
126
120
  end
127
121
 
128
122
  ##
129
123
  # @return [Boolean] whether the schema validation succeeded
130
124
  def valid_schema?
131
- validation_errors.nil? || validation_errors.empty?
125
+ validation_issues.nil? || validation_issues.empty?
132
126
  end
133
127
 
134
128
  ##
@@ -138,8 +132,8 @@ module Html2rss
138
132
  end
139
133
 
140
134
  ##
141
- # @return [Hash{Symbol => Object}] hash representation
142
- def to_h # rubocop:disable Metrics/MethodLength
135
+ # @return [Hash{Symbol => Object}] public test hash; +response_body+ stays off this wire
136
+ def to_h # rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- wire serialization omits response_body
143
137
  {
144
138
  success:,
145
139
  item_count:,
@@ -148,7 +142,7 @@ module Html2rss
148
142
  channel_url:,
149
143
  strategy_used:,
150
144
  duration_seconds:,
151
- validation_errors:,
145
+ validation_issues: validation_issues&.map(&:to_h),
152
146
  error_message:,
153
147
  failure_kind: failure_kind&.to_sym,
154
148
  rss:,
@@ -169,92 +163,113 @@ module Html2rss
169
163
  # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
170
164
  # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
171
165
  # @return [Html2rss::Test::Result]
172
- def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
166
+ def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
173
167
  strict_quality: false, compare_enhance: false)
174
168
  raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
175
- return validation_failure_result(validation.errors.to_h, raw_config) unless validation.success?
169
+ return validation_failure_result(validation, raw_config) unless validation.success?
176
170
 
171
+ config = build_test_config(raw_config, strategy, params)
172
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
173
+ execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
174
+ rescue StandardError => error
175
+ duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
176
+ execution_failure_result(error, raw_config, duration)
177
+ end
178
+
179
+ def build_test_config(raw_config, strategy, params)
177
180
  raw_config[:strategy] = strategy.to_sym if strategy
178
181
  raw_config[:params] = params if params&.any?
182
+ Config.from_hash(raw_config)
183
+ end
184
+ private_class_method :build_test_config
185
+
186
+ def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
187
+ quality_report_for(
188
+ feed_result.to_rss.items,
189
+ channel_url: raw_config.dig(:channel, :url).to_s,
190
+ config:,
191
+ feed_result:,
192
+ pipeline_outcome:
193
+ )
194
+ end
195
+ private_class_method :compile_quality_report
179
196
 
180
- started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
181
- begin
182
- feed_result, pipeline_outcome = extract_feed(raw_config)
183
- duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
184
-
185
- rss_doc = feed_result.to_rss
186
- rss_xml = rss_doc.to_s
187
- item_count = rss_doc.items.size
188
- sample_items = extract_samples(rss_doc.items)
189
- quality_report = quality_report_for(
190
- rss_doc.items,
191
- channel_url: raw_config.dig(:channel, :url).to_s,
192
- raw_config:,
193
- feed_result:,
194
- pipeline_outcome:
195
- )
196
- enhance_compare = build_enhance_compare(raw_config, pipeline_outcome) if compare_enhance
197
-
198
- channel_title = feed_result.channel_title
199
- channel_url = raw_config.dig(:channel, :url).to_s
200
- strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] || :faraday
201
- min_items_passed = item_count >= min_items
202
- quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
203
- passed = min_items_passed && !quality_failed
204
- failure_kind, error_message = outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:,
205
- quality_report:)
206
-
207
- log_strict_quality_failure(failure_kind, item_count) if quality_failed
208
-
209
- Result.new(
210
- success: passed,
211
- item_count:,
212
- sample_items:,
213
- channel_title:,
214
- channel_url:,
215
- strategy_used:,
216
- duration_seconds: duration.round(3),
217
- validation_errors: nil,
218
- error_message:,
219
- failure_kind:,
220
- rss: passed ? rss_xml : nil,
221
- quality_report:,
222
- enhance_compare:
223
- )
224
- rescue StandardError => error
225
- duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
226
- execution_failure_result(error, raw_config, duration)
227
- end
197
+ def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
198
+ compare_enhance:)
199
+ feed_result, pipeline_outcome = extract_feed(raw_config)
200
+ duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
201
+ quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
202
+ enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
203
+
204
+ build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
205
+ min_items:, strict_quality:, response_body: pipeline_outcome&.response&.body)
206
+ end
207
+ private_class_method :execute_timed_pipeline
208
+
209
+ def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
210
+ min_items:, strict_quality:, response_body:)
211
+ rss_doc = feed_result.to_rss
212
+ outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
213
+ strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
214
+ RequestService.default_strategy_name
215
+
216
+ Result.new(
217
+ success: outcome.passed,
218
+ item_count: rss_doc.items.size,
219
+ sample_items: extract_samples(rss_doc.items),
220
+ channel_title: feed_result.channel_title,
221
+ channel_url: raw_config.dig(:channel, :url).to_s,
222
+ strategy_used:,
223
+ duration_seconds: duration.round(3),
224
+ validation_issues: nil,
225
+ error_message: outcome.error_message,
226
+ failure_kind: outcome.failure_kind,
227
+ rss: outcome.passed ? rss_doc.to_s : nil,
228
+ quality_report:,
229
+ enhance_compare:,
230
+ response_body:
231
+ )
228
232
  end
233
+ private_class_method :build_test_result
229
234
 
230
235
  def extract_samples(items, limit: 3)
231
236
  items.first(limit).map do |item|
237
+ article = feed_item_as_article(item)
232
238
  {
233
- title: item.title.to_s.strip,
234
- url: (item.respond_to?(:link) ? item.link : item.url).to_s,
235
- published_at: (item.respond_to?(:pubDate) ? item.pubDate : item.published_at)
239
+ title: article.title.to_s.strip,
240
+ url: article.url.to_s,
241
+ published_at: article.published_at
236
242
  }
237
243
  end
238
244
  end
239
245
  private_class_method :extract_samples
240
246
 
247
+ def feed_item_as_article(item)
248
+ case item
249
+ when Article then item
250
+ else
251
+ Article.new(title: item.title, url: item.link, published_at: item.pubDate)
252
+ end
253
+ end
254
+ private_class_method :feed_item_as_article
255
+
241
256
  ##
242
257
  # Builds a ship-quality report for RSS items (shared by test and MCP apply).
243
258
  #
244
259
  # @param items [Array]
245
260
  # @param channel_url [String]
246
- # @param raw_config [Hash]
261
+ # @param config [Html2rss::Config]
247
262
  # @param feed_result [Html2rss::FeedResult]
248
263
  # @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
249
264
  # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
250
265
  # @return [QualityReport]
251
- def quality_report_for(items, channel_url:, raw_config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
266
+ def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
252
267
  probe_native_feed: true)
253
268
  audit = AutoSource::Cleanup.audit_feed_items(items)
254
- native_feed = probe_native_feed ? probe_native_feed_url(channel_url, raw_config)&.to_s : nil
269
+ native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
255
270
  report = QualityReport.from_audit(audit, native_feed:)
256
271
  report = append_url_mismatch_warning(report, channel_url, feed_result)
257
- merge_enhance_audit(report, raw_config, pipeline_outcome)
272
+ merge_enhance_audit(report, config, pipeline_outcome)
258
273
  end
259
274
 
260
275
  def extract_feed(raw_config)
@@ -263,18 +278,15 @@ module Html2rss
263
278
  end
264
279
  private_class_method :extract_feed
265
280
 
266
- def enhance_enabled?(raw_config)
267
- config = Config.from_hash(raw_config)
281
+ def enhance_enabled?(config)
268
282
  return false unless config.selectors
269
283
 
270
284
  !!config.selectors.dig(:items, :enhance)
271
285
  end
272
286
  private_class_method :enhance_enabled?
273
287
 
274
- def build_enhance_compare(raw_config, pipeline_outcome)
288
+ def build_enhance_compare(config, pipeline_outcome)
275
289
  return nil unless pipeline_outcome
276
-
277
- config = Config.from_hash(raw_config)
278
290
  return nil unless config.selectors
279
291
 
280
292
  EnhanceAudit.compare(
@@ -285,10 +297,8 @@ module Html2rss
285
297
  end
286
298
  private_class_method :build_enhance_compare
287
299
 
288
- def merge_enhance_audit(report, raw_config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
289
- return report unless pipeline_outcome && enhance_enabled?(raw_config)
290
-
291
- config = Config.from_hash(raw_config)
300
+ def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
301
+ return report unless pipeline_outcome && enhance_enabled?(config)
292
302
  return report unless config.selectors
293
303
 
294
304
  slice = EnhanceAudit.probe(
@@ -349,67 +359,11 @@ module Html2rss
349
359
  end
350
360
  private_class_method :url_mismatch?
351
361
 
352
- def quality_failure?(quality_report)
353
- metrics = quality_report.metrics
354
- item_count = metrics[:item_count]
355
- return false if item_count.zero?
356
-
357
- duplicate_urls_failure?(metrics) || junk_title_ratio_failure?(metrics) || short_title_failure?(metrics)
358
- end
359
- private_class_method :quality_failure?
360
-
361
- def duplicate_urls_failure?(metrics)
362
- metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
363
- end
364
- private_class_method :duplicate_urls_failure?
365
-
366
- def junk_title_ratio_failure?(metrics)
367
- metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
368
- end
369
- private_class_method :junk_title_ratio_failure?
370
-
371
- def short_title_failure?(metrics)
372
- metrics[:short_title_count].positive?
373
- end
374
- private_class_method :short_title_failure?
375
-
376
- def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
377
- return [nil, nil] if min_items_passed && !quality_failed
378
- unless min_items_passed
379
- return [FailureKind.coerce(:min_items),
380
- "Extracted #{item_count} items (minimum required: #{min_items})"]
381
- end
382
-
383
- [FailureKind.coerce(:quality), quality_failure_message(quality_report)]
384
- end
385
- private_class_method :outcome_failure
386
-
387
- def quality_failure_message(quality_report)
388
- reasons = quality_failure_reasons(quality_report.metrics)
389
- "Feed quality check failed (#{reasons.join(', ')})"
390
- end
391
- private_class_method :quality_failure_message
392
-
393
- def quality_failure_reasons(metrics)
394
- reasons = []
395
- reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
396
- reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
397
- reasons << 'short_titles' if short_title_failure?(metrics)
398
- reasons
399
- end
400
- private_class_method :quality_failure_reasons
401
-
402
- def log_strict_quality_failure(failure_kind, item_count)
403
- Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
404
- end
405
- private_class_method :log_strict_quality_failure
406
-
407
- def probe_native_feed_url(channel_url, raw_config)
362
+ def probe_native_feed_url(channel_url, config)
408
363
  return nil if channel_url.to_s.empty?
409
364
 
410
- config = Config.from_hash(raw_config)
411
365
  resources = FeedPipeline::RuntimePolicy.resources_for(config)
412
- strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(raw_config[:strategy])
366
+ strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(config.strategy)
413
367
  session = RequestSession.build(
414
368
  config:, strategy:, budget: resources.budget, policy: resources.policy
415
369
  )
@@ -419,7 +373,7 @@ module Html2rss
419
373
  end
420
374
  private_class_method :probe_native_feed_url
421
375
 
422
- def validation_failure_result(errors, raw_config) # rubocop:disable Metrics/MethodLength
376
+ def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
423
377
  Result.new(
424
378
  success: false,
425
379
  item_count: 0,
@@ -428,9 +382,9 @@ module Html2rss
428
382
  channel_url: raw_config.dig(:channel, :url),
429
383
  strategy_used: raw_config[:strategy],
430
384
  duration_seconds: 0.0,
431
- validation_errors: errors,
385
+ validation_issues: report.issues,
432
386
  error_message: 'Configuration schema validation failed',
433
- failure_kind: FailureKind.coerce(:schema),
387
+ failure_kind: FailureKind[:schema],
434
388
  rss: nil,
435
389
  quality_report: nil
436
390
  )
@@ -446,9 +400,9 @@ module Html2rss
446
400
  channel_url: raw_config.dig(:channel, :url),
447
401
  strategy_used: raw_config[:strategy],
448
402
  duration_seconds: duration.round(3),
449
- validation_errors: nil,
403
+ validation_issues: nil,
450
404
  error_message: "#{error.class}: #{error.message}",
451
- failure_kind: FailureKind.coerce(:execution),
405
+ failure_kind: FailureKind[:execution],
452
406
  rss: nil,
453
407
  quality_report: nil
454
408
  )
data/lib/html2rss/url.rb CHANGED
@@ -42,14 +42,21 @@ module Html2rss
42
42
  url = Addressable::URI.parse(relative_url.to_s.strip)
43
43
  return new(url) if url.absolute?
44
44
 
45
- base_uri = Addressable::URI.parse(base_url.to_s)
46
- base_uri.path = '/' if base_uri.path.empty?
47
-
48
- new(base_uri.join(url).normalize)
45
+ base_url.is_a?(self) ? base_url.join_uri(url) : join_parsed_base(base_url, url)
49
46
  rescue Addressable::URI::InvalidURIError
50
47
  raise ArgumentError, 'URL could not be parsed'
51
48
  end
52
49
 
50
+ # @param base_url [String]
51
+ # @param url [Addressable::URI]
52
+ # @return [Url]
53
+ def self.join_parsed_base(base_url, url)
54
+ uri = Addressable::URI.parse(base_url.to_s)
55
+ uri.path = '/' if uri.path.empty?
56
+ new(uri.join(url).normalize)
57
+ end
58
+ private_class_method :join_parsed_base
59
+
53
60
  ##
54
61
  # Creates a URL by sanitizing a raw URL string.
55
62
  # Removes spaces and extracts the first valid URL from the string.
@@ -305,6 +312,20 @@ module Html2rss
305
312
  # @return [String] the debug representation
306
313
  def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
307
314
 
315
+ ##
316
+ # Resolves +relative_uri+ against this URL's already-parsed Addressable URI.
317
+ #
318
+ # @param relative_uri [Addressable::URI]
319
+ # @return [Url]
320
+ def join_uri(relative_uri)
321
+ base = @uri
322
+ if base.path.empty?
323
+ base = base.dup
324
+ base.path = '/'
325
+ end
326
+ self.class.new(base.join(relative_uri).normalize)
327
+ end
328
+
308
329
  protected
309
330
 
310
331
  # Scheme, authority, slash-stripped path, and query (fragment omitted).
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Html2rss
4
4
  # Current application version.
5
- VERSION = '0.29.1'
5
+ VERSION = '0.31.0'
6
6
  public_constant :VERSION
7
7
  end
data/lib/html2rss.rb CHANGED
@@ -11,20 +11,30 @@ loader.inflector.inflect(
11
11
  loader.setup
12
12
 
13
13
  require 'logger'
14
- require 'forwardable'
15
14
  require 'html2rss/defaults'
16
15
 
17
16
  ##
18
17
  # The Html2rss namespace.
19
18
  module Html2rss # rubocop:disable Metrics/ModuleLength
20
19
  ##
21
- # The logger instance.
20
+ # Forwards debug/info/warn/error to {Html2rss.logger}.
22
21
  module Log
23
22
  class << self
24
- extend Forwardable
23
+ ##
24
+ # @return [true]
25
+ def debug(...) = Html2rss.logger.debug(...)
25
26
 
26
- def_delegator 'Html2rss', :logger
27
- def_delegators :logger, :debug, :info, :warn, :error, :fatal, :unknown, :level, :level=, :formatter, :formatter=
27
+ ##
28
+ # @return [true]
29
+ def info(...) = Html2rss.logger.info(...)
30
+
31
+ ##
32
+ # @return [true]
33
+ def warn(...) = Html2rss.logger.warn(...)
34
+
35
+ ##
36
+ # @return [true]
37
+ def error(...) = Html2rss.logger.error(...)
28
38
  end
29
39
  end
30
40
 
@@ -33,7 +43,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
33
43
  # Golden path step 1 (optional); use {.recon} for verdict and native_feed.
34
44
  #
35
45
  # @param url [String] source page URL
36
- # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
46
+ # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
37
47
  # @param deep [Boolean] when true and strategy is :auto, one Botasaurus hop if configured
38
48
  # @return [Html2rss::PageRecon::Diagnostics::Report]
39
49
  def self.inspect(url, strategy: :auto, deep: false, **)
@@ -45,7 +55,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
45
55
  # Golden path step 2 (optional); adds verdict beyond {.inspect}.
46
56
  #
47
57
  # @param url [String, Html2rss::Url] source page URL
48
- # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
58
+ # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
49
59
  # @return [Html2rss::Recon::Result]
50
60
  def self.recon(url, strategy: :auto, **)
51
61
  Recon.call(url, strategy:, **)
@@ -56,7 +66,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
56
66
  # Golden path step 3.
57
67
  #
58
68
  # @param url [String] source page URL
59
- # @param strategy [Symbol] request strategy (+:auto+, +:faraday+, +:botasaurus+)
69
+ # @param strategy [Symbol] request strategy (+:auto+, +:default+, +:botasaurus+)
60
70
  # @option options [String, nil] :items_selector optional CSS selector hint for items
61
71
  # @option options [Array<String>, nil] :topics optional directory topics override
62
72
  # @option options [String, nil] :title optional title override
@@ -79,7 +89,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
79
89
  # @param config_input [Hash, String]
80
90
  # @param feed_name [String, nil]
81
91
  # @param params [Hash]
82
- # @return [Dry::Validation::Result, Html2rss::Config::ValidationResult]
92
+ # @return [Html2rss::Config::ValidationReport]
83
93
  def self.validate(config_input, feed_name = nil, params: {})
84
94
  _raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
85
95
  validation
@@ -140,7 +150,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
140
150
  # Scrapes multiple URLs in parallel using auto-source article discovery.
141
151
  #
142
152
  # @param urls [Enumerable<String>] list of URLs to scrape
143
- # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
153
+ # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
144
154
  # @param limit [Integer] max articles to keep per URL (default: 10)
145
155
  # @param concurrency [Integer] max worker threads (default: 5)
146
156
  # @return [Html2rss::Batch::BatchResult]
@@ -152,7 +162,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
152
162
  # Inspects multiple URLs in parallel with per-URL error isolation.
153
163
  #
154
164
  # @param urls [Enumerable<String>] list of URLs to inspect
155
- # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
165
+ # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
156
166
  # @param concurrency [Integer] max worker threads (default: 5)
157
167
  # @return [Html2rss::Batch::BatchResult]
158
168
  def self.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
@@ -163,7 +173,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
163
173
  # Runs recon across multiple URLs in parallel with per-URL error isolation.
164
174
  #
165
175
  # @param urls [Enumerable<String>] list of URLs to recon
166
- # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
176
+ # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
167
177
  # @param concurrency [Integer] max worker threads (default: 5)
168
178
  # @option options [String, nil] :cache_dir optional HTML cache directory
169
179
  # @return [Html2rss::Batch::BatchResult]
@@ -179,7 +189,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
179
189
  # @param pretty [Boolean] whether to pretty-print JSON
180
190
  # @return [String]
181
191
  def self.schema_json(pretty: true)
182
- Config.json_schema_json(pretty:)
192
+ Config::Schema.json_schema_json(pretty:)
183
193
  end
184
194
 
185
195
  ##
@@ -319,26 +329,24 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
319
329
  end
320
330
 
321
331
  ##
322
- # @return [Object] the logger
332
+ # @return [Logger] the logger
323
333
  def logger
324
334
  defaults.logger
325
335
  end
326
336
 
327
337
  ##
328
- # @param logger [Object] the new logger
338
+ # @param logger [Logger] the new logger
329
339
  def logger=(logger)
330
340
  configure { |config| config.logger = logger }
331
341
  end
332
342
 
333
- private
334
-
335
343
  ##
336
344
  # Resets the global defaults (mainly for testing).
337
345
  #
338
346
  # @return [void]
339
347
  def reset_defaults!
340
348
  @defaults = nil
341
- logger.level = defaults.log_level if logger.respond_to?(:level=)
349
+ logger.level = defaults.log_level
342
350
  end
343
351
  end
344
352
 
@@ -362,7 +370,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
362
370
  end
363
371
  end
364
372
 
365
- logger.level = defaults.log_level if logger.respond_to?(:level=)
373
+ logger.level = defaults.log_level
366
374
  end
367
375
 
368
376
  loader.eager_load
@@ -7,10 +7,10 @@ require_relative '../html2rss'
7
7
  namespace :config do
8
8
  desc 'Generate config JSON schema'
9
9
  task :schema do
10
- destination = Html2rss::Config.schema_path
10
+ destination = Html2rss::Config::Schema.path
11
11
 
12
12
  FileUtils.mkdir_p(File.dirname(destination))
13
- File.write(destination, "#{Html2rss::Config.json_schema_json}\n")
13
+ File.write(destination, "#{Html2rss::Config::Schema.json_schema_json}\n")
14
14
 
15
15
  puts "Generated config schema at #{destination}"
16
16
  end