html2rss 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/lib/html2rss/article/deduplicator.rb +2 -4
  4. data/lib/html2rss/article.rb +16 -39
  5. data/lib/html2rss/auto_source/README.md +1 -1
  6. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  7. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  8. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  10. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  11. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  13. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  14. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  15. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  16. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  17. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper.rb +4 -14
  19. data/lib/html2rss/auto_source/segment.rb +7 -4
  20. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  21. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  22. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  23. data/lib/html2rss/auto_source.rb +2 -2
  24. data/lib/html2rss/batch.rb +3 -1
  25. data/lib/html2rss/capture/README.md +6 -4
  26. data/lib/html2rss/capture.rb +67 -35
  27. data/lib/html2rss/channel.rb +3 -3
  28. data/lib/html2rss/cli/probe_view.rb +5 -3
  29. data/lib/html2rss/cli/render.rb +6 -5
  30. data/lib/html2rss/cli/validate.rb +28 -27
  31. data/lib/html2rss/cli.rb +1 -1
  32. data/lib/html2rss/config/issue_mapper.rb +108 -0
  33. data/lib/html2rss/config/request_headers.rb +2 -3
  34. data/lib/html2rss/config/schema.rb +16 -1
  35. data/lib/html2rss/config/selectors_validator.rb +92 -70
  36. data/lib/html2rss/config/validation_issue.rb +60 -0
  37. data/lib/html2rss/config/validation_report.rb +55 -0
  38. data/lib/html2rss/config/validator.rb +10 -6
  39. data/lib/html2rss/config.rb +35 -113
  40. data/lib/html2rss/defaults.rb +42 -30
  41. data/lib/html2rss/doctor/botasaurus.rb +6 -4
  42. data/lib/html2rss/doctor.rb +8 -0
  43. data/lib/html2rss/error.rb +1 -1
  44. data/lib/html2rss/feed_builder/rss.rb +9 -14
  45. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  46. data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
  47. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  48. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  49. data/lib/html2rss/feed_resolution/options.rb +3 -1
  50. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  51. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  52. data/lib/html2rss/html/article_extractor.rb +2 -4
  53. data/lib/html2rss/html/feed_link.rb +3 -1
  54. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  55. data/lib/html2rss/html/navigator.rb +10 -10
  56. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  57. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  58. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  59. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  60. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  61. data/lib/html2rss/mcp/README.md +2 -0
  62. data/lib/html2rss/mcp/config_argument.rb +3 -1
  63. data/lib/html2rss/mcp/contract.rb +6 -20
  64. data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
  65. data/lib/html2rss/mcp/outcome.rb +36 -43
  66. data/lib/html2rss/mcp/server/tools.rb +111 -131
  67. data/lib/html2rss/mcp/server.rb +26 -18
  68. data/lib/html2rss/page_recon/README.md +5 -5
  69. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  70. data/lib/html2rss/page_recon.rb +9 -5
  71. data/lib/html2rss/recon.rb +19 -30
  72. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  73. data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
  74. data/lib/html2rss/request_service/compressed_body.rb +4 -1
  75. data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
  76. data/lib/html2rss/request_service/network_guard.rb +7 -5
  77. data/lib/html2rss/request_service/response.rb +1 -2
  78. data/lib/html2rss/request_service/strategy.rb +3 -8
  79. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  80. data/lib/html2rss/scoring/engine.rb +8 -8
  81. data/lib/html2rss/scoring/observation.rb +5 -3
  82. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  83. data/lib/html2rss/scrape_target.rb +3 -1
  84. data/lib/html2rss/selector_candidates.rb +234 -0
  85. data/lib/html2rss/selectors/README.md +64 -0
  86. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  87. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  88. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  89. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  90. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  91. data/lib/html2rss/selectors/extractors.rb +33 -20
  92. data/lib/html2rss/selectors/item_env.rb +31 -0
  93. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  94. data/lib/html2rss/selectors/option_spec.rb +140 -0
  95. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  96. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  97. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  98. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  99. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  100. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  101. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  102. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  103. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  104. data/lib/html2rss/selectors/post_processors.rb +6 -5
  105. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  106. data/lib/html2rss/selectors/step_env.rb +24 -0
  107. data/lib/html2rss/selectors.rb +231 -142
  108. data/lib/html2rss/sst/attrs.rb +5 -3
  109. data/lib/html2rss/sst/document.rb +3 -1
  110. data/lib/html2rss/sst/index.rb +3 -1
  111. data/lib/html2rss/sst/node.rb +29 -10
  112. data/lib/html2rss/sst/normalizer.rb +12 -16
  113. data/lib/html2rss/sst/tags.rb +7 -7
  114. data/lib/html2rss/status.rb +13 -8
  115. data/lib/html2rss/surface_category.rb +14 -32
  116. data/lib/html2rss/syndication/parser.rb +2 -2
  117. data/lib/html2rss/test/enhance_audit.rb +1 -3
  118. data/lib/html2rss/test/policy.rb +106 -0
  119. data/lib/html2rss/test.rb +126 -173
  120. data/lib/html2rss/url.rb +25 -4
  121. data/lib/html2rss/version.rb +1 -1
  122. data/lib/html2rss.rb +21 -13
  123. data/lib/tasks/config_schema.rake +2 -2
  124. data/schema/html2rss-config.schema.json +11 -8
  125. metadata +15 -6
  126. data/lib/html2rss/selectors/item_scope.rb +0 -30
data/lib/html2rss/test.rb CHANGED
@@ -12,27 +12,14 @@ module Html2rss
12
12
  ##
13
13
  # Closed failure classification for a failed test run.
14
14
  class FailureKind
15
- # Closed set of test failure wire names.
16
- NAMES = Set[:schema, :execution, :min_items, :quality].freeze
17
-
18
- class << self
19
- ##
20
- # @param value [FailureKind, Symbol, String]
21
- # @return [FailureKind]
22
- def coerce(value)
23
- return value if value.is_a?(self)
24
-
25
- new(name: value.to_sym)
26
- end
27
- end
15
+ # Frozen name → instance table.
16
+ ALL = %i[schema execution min_items quality].to_h { |name| [name, new(name:)] }.freeze
17
+ private_class_method :new
28
18
 
29
19
  ##
30
- # @param name [Symbol]
31
- def initialize(name:)
32
- raise ArgumentError, "unknown failure kind: #{name.inspect}" unless NAMES.include?(name)
33
-
34
- super
35
- end
20
+ # @param name [Symbol, String]
21
+ # @return [FailureKind]
22
+ def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown failure kind: #{name.inspect}" }
36
23
 
37
24
  ##
38
25
  # @return [Boolean]
@@ -59,19 +46,11 @@ module Html2rss
59
46
  def to_s = name.to_s
60
47
  end
61
48
 
49
+ QualityReport = Data.define(:warnings, :metrics, :native_feed, :defer_reason)
50
+
62
51
  ##
63
52
  # Ship-quality audit summary for a configuration test (warn-only).
64
- QualityReport = Data.define(:warnings, :metrics, :native_feed, :defer_reason) do
65
- ##
66
- # @return [Hash{Symbol => Object}]
67
- def to_h
68
- {
69
- warnings: warnings.map(&:to_s),
70
- metrics:,
71
- **(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
72
- }.compact
73
- end
74
-
53
+ class QualityReport
75
54
  ##
76
55
  # @param audit [Html2rss::AutoSource::Cleanup::AuditResult]
77
56
  # @param native_feed [String, nil]
@@ -85,11 +64,18 @@ module Html2rss
85
64
  end
86
65
  new(warnings: report_warnings.freeze, metrics: audit.metrics, native_feed:, defer_reason:)
87
66
  end
67
+
68
+ ##
69
+ # @return [Hash{Symbol => Object}]
70
+ def to_h
71
+ {
72
+ warnings: warnings.map(&:to_s),
73
+ metrics:,
74
+ **(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
75
+ }.compact
76
+ end
88
77
  end
89
78
 
90
- ##
91
- # Immutable outcome of a configuration test. Success carries +rss+ XML from the
92
- # first live extraction; failures carry a typed {FailureKind}.
93
79
  Result = Data.define(
94
80
  :success,
95
81
  :item_count,
@@ -98,13 +84,20 @@ module Html2rss
98
84
  :channel_url,
99
85
  :strategy_used,
100
86
  :duration_seconds,
101
- :validation_errors,
87
+ :validation_issues,
102
88
  :error_message,
103
89
  :failure_kind,
104
90
  :rss,
105
91
  :quality_report,
106
- :enhance_compare
107
- ) do
92
+ :enhance_compare,
93
+ :response_body
94
+ )
95
+
96
+ ##
97
+ # Immutable outcome of a configuration test. Success carries +rss+ XML from the
98
+ # first live extraction; failures carry a typed {FailureKind}.
99
+ # +response_body+ is a copy of the fetched page for in-process readers and stays off {#to_h}.
100
+ class Result
108
101
  ##
109
102
  # @param success [Boolean]
110
103
  # @param item_count [Integer]
@@ -113,22 +106,23 @@ module Html2rss
113
106
  # @param channel_url [String, nil]
114
107
  # @param strategy_used [Symbol, nil]
115
108
  # @param duration_seconds [Float]
116
- # @param validation_errors [Hash, nil]
109
+ # @param validation_issues [Array<Html2rss::Config::ValidationIssue>, nil]
117
110
  # @param error_message [String, nil]
118
111
  # @param failure_kind [FailureKind, nil]
119
112
  # @param rss [String, nil]
120
113
  # @param quality_report [QualityReport, nil]
121
114
  # @param enhance_compare [Hash, nil]
115
+ # @param response_body [String, nil] fetched page HTML copied from the pipeline response
122
116
  def initialize(success:, item_count:, sample_items:, channel_title:, channel_url:, # rubocop:disable Metrics/ParameterLists
123
- strategy_used:, duration_seconds:, validation_errors:, error_message:,
124
- failure_kind:, rss:, quality_report: nil, enhance_compare: nil)
117
+ strategy_used:, duration_seconds:, validation_issues:, error_message:,
118
+ failure_kind:, rss:, quality_report: nil, enhance_compare: nil, response_body: nil)
125
119
  super
126
120
  end
127
121
 
128
122
  ##
129
123
  # @return [Boolean] whether the schema validation succeeded
130
124
  def valid_schema?
131
- validation_errors.nil? || validation_errors.empty?
125
+ validation_issues.nil? || validation_issues.empty?
132
126
  end
133
127
 
134
128
  ##
@@ -138,8 +132,8 @@ module Html2rss
138
132
  end
139
133
 
140
134
  ##
141
- # @return [Hash{Symbol => Object}] hash representation
142
- def to_h # rubocop:disable Metrics/MethodLength
135
+ # @return [Hash{Symbol => Object}] public test hash; +response_body+ stays off this wire
136
+ def to_h # rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- wire serialization omits response_body
143
137
  {
144
138
  success:,
145
139
  item_count:,
@@ -148,7 +142,7 @@ module Html2rss
148
142
  channel_url:,
149
143
  strategy_used:,
150
144
  duration_seconds:,
151
- validation_errors:,
145
+ validation_issues: validation_issues&.map(&:to_h),
152
146
  error_message:,
153
147
  failure_kind: failure_kind&.to_sym,
154
148
  rss:,
@@ -169,93 +163,113 @@ module Html2rss
169
163
  # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
170
164
  # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
171
165
  # @return [Html2rss::Test::Result]
172
- def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
166
+ def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
173
167
  strict_quality: false, compare_enhance: false)
174
168
  raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
175
- return validation_failure_result(validation.errors.to_h, raw_config) unless validation.success?
169
+ return validation_failure_result(validation, raw_config) unless validation.success?
176
170
 
171
+ config = build_test_config(raw_config, strategy, params)
172
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
173
+ execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
174
+ rescue StandardError => error
175
+ duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
176
+ execution_failure_result(error, raw_config, duration)
177
+ end
178
+
179
+ def build_test_config(raw_config, strategy, params)
177
180
  raw_config[:strategy] = strategy.to_sym if strategy
178
181
  raw_config[:params] = params if params&.any?
182
+ Config.from_hash(raw_config)
183
+ end
184
+ private_class_method :build_test_config
185
+
186
+ def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
187
+ quality_report_for(
188
+ feed_result.to_rss.items,
189
+ channel_url: raw_config.dig(:channel, :url).to_s,
190
+ config:,
191
+ feed_result:,
192
+ pipeline_outcome:
193
+ )
194
+ end
195
+ private_class_method :compile_quality_report
179
196
 
180
- started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
181
- begin
182
- feed_result, pipeline_outcome = extract_feed(raw_config)
183
- duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
184
-
185
- rss_doc = feed_result.to_rss
186
- rss_xml = rss_doc.to_s
187
- item_count = rss_doc.items.size
188
- sample_items = extract_samples(rss_doc.items)
189
- quality_report = quality_report_for(
190
- rss_doc.items,
191
- channel_url: raw_config.dig(:channel, :url).to_s,
192
- raw_config:,
193
- feed_result:,
194
- pipeline_outcome:
195
- )
196
- enhance_compare = build_enhance_compare(raw_config, pipeline_outcome) if compare_enhance
197
-
198
- channel_title = feed_result.channel_title
199
- channel_url = raw_config.dig(:channel, :url).to_s
200
- strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
201
- RequestService.default_strategy_name
202
- min_items_passed = item_count >= min_items
203
- quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
204
- passed = min_items_passed && !quality_failed
205
- failure_kind, error_message = outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:,
206
- quality_report:)
207
-
208
- log_strict_quality_failure(failure_kind, item_count) if quality_failed
209
-
210
- Result.new(
211
- success: passed,
212
- item_count:,
213
- sample_items:,
214
- channel_title:,
215
- channel_url:,
216
- strategy_used:,
217
- duration_seconds: duration.round(3),
218
- validation_errors: nil,
219
- error_message:,
220
- failure_kind:,
221
- rss: passed ? rss_xml : nil,
222
- quality_report:,
223
- enhance_compare:
224
- )
225
- rescue StandardError => error
226
- duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
227
- execution_failure_result(error, raw_config, duration)
228
- end
197
+ def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
198
+ compare_enhance:)
199
+ feed_result, pipeline_outcome = extract_feed(raw_config)
200
+ duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
201
+ quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
202
+ enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
203
+
204
+ build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
205
+ min_items:, strict_quality:, response_body: pipeline_outcome&.response&.body)
206
+ end
207
+ private_class_method :execute_timed_pipeline
208
+
209
+ def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
210
+ min_items:, strict_quality:, response_body:)
211
+ rss_doc = feed_result.to_rss
212
+ outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
213
+ strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
214
+ RequestService.default_strategy_name
215
+
216
+ Result.new(
217
+ success: outcome.passed,
218
+ item_count: rss_doc.items.size,
219
+ sample_items: extract_samples(rss_doc.items),
220
+ channel_title: feed_result.channel_title,
221
+ channel_url: raw_config.dig(:channel, :url).to_s,
222
+ strategy_used:,
223
+ duration_seconds: duration.round(3),
224
+ validation_issues: nil,
225
+ error_message: outcome.error_message,
226
+ failure_kind: outcome.failure_kind,
227
+ rss: outcome.passed ? rss_doc.to_s : nil,
228
+ quality_report:,
229
+ enhance_compare:,
230
+ response_body:
231
+ )
229
232
  end
233
+ private_class_method :build_test_result
230
234
 
231
235
  def extract_samples(items, limit: 3)
232
236
  items.first(limit).map do |item|
237
+ article = feed_item_as_article(item)
233
238
  {
234
- title: item.title.to_s.strip,
235
- url: (item.respond_to?(:link) ? item.link : item.url).to_s,
236
- published_at: (item.respond_to?(:pubDate) ? item.pubDate : item.published_at)
239
+ title: article.title.to_s.strip,
240
+ url: article.url.to_s,
241
+ published_at: article.published_at
237
242
  }
238
243
  end
239
244
  end
240
245
  private_class_method :extract_samples
241
246
 
247
+ def feed_item_as_article(item)
248
+ case item
249
+ when Article then item
250
+ else
251
+ Article.new(title: item.title, url: item.link, published_at: item.pubDate)
252
+ end
253
+ end
254
+ private_class_method :feed_item_as_article
255
+
242
256
  ##
243
257
  # Builds a ship-quality report for RSS items (shared by test and MCP apply).
244
258
  #
245
259
  # @param items [Array]
246
260
  # @param channel_url [String]
247
- # @param raw_config [Hash]
261
+ # @param config [Html2rss::Config]
248
262
  # @param feed_result [Html2rss::FeedResult]
249
263
  # @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
250
264
  # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
251
265
  # @return [QualityReport]
252
- def quality_report_for(items, channel_url:, raw_config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
266
+ def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
253
267
  probe_native_feed: true)
254
268
  audit = AutoSource::Cleanup.audit_feed_items(items)
255
- native_feed = probe_native_feed ? probe_native_feed_url(channel_url, raw_config)&.to_s : nil
269
+ native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
256
270
  report = QualityReport.from_audit(audit, native_feed:)
257
271
  report = append_url_mismatch_warning(report, channel_url, feed_result)
258
- merge_enhance_audit(report, raw_config, pipeline_outcome)
272
+ merge_enhance_audit(report, config, pipeline_outcome)
259
273
  end
260
274
 
261
275
  def extract_feed(raw_config)
@@ -264,18 +278,15 @@ module Html2rss
264
278
  end
265
279
  private_class_method :extract_feed
266
280
 
267
- def enhance_enabled?(raw_config)
268
- config = Config.from_hash(raw_config)
281
+ def enhance_enabled?(config)
269
282
  return false unless config.selectors
270
283
 
271
284
  !!config.selectors.dig(:items, :enhance)
272
285
  end
273
286
  private_class_method :enhance_enabled?
274
287
 
275
- def build_enhance_compare(raw_config, pipeline_outcome)
288
+ def build_enhance_compare(config, pipeline_outcome)
276
289
  return nil unless pipeline_outcome
277
-
278
- config = Config.from_hash(raw_config)
279
290
  return nil unless config.selectors
280
291
 
281
292
  EnhanceAudit.compare(
@@ -286,10 +297,8 @@ module Html2rss
286
297
  end
287
298
  private_class_method :build_enhance_compare
288
299
 
289
- def merge_enhance_audit(report, raw_config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
290
- return report unless pipeline_outcome && enhance_enabled?(raw_config)
291
-
292
- config = Config.from_hash(raw_config)
300
+ def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
301
+ return report unless pipeline_outcome && enhance_enabled?(config)
293
302
  return report unless config.selectors
294
303
 
295
304
  slice = EnhanceAudit.probe(
@@ -350,67 +359,11 @@ module Html2rss
350
359
  end
351
360
  private_class_method :url_mismatch?
352
361
 
353
- def quality_failure?(quality_report)
354
- metrics = quality_report.metrics
355
- item_count = metrics[:item_count]
356
- return false if item_count.zero?
357
-
358
- duplicate_urls_failure?(metrics) || junk_title_ratio_failure?(metrics) || short_title_failure?(metrics)
359
- end
360
- private_class_method :quality_failure?
361
-
362
- def duplicate_urls_failure?(metrics)
363
- metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
364
- end
365
- private_class_method :duplicate_urls_failure?
366
-
367
- def junk_title_ratio_failure?(metrics)
368
- metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
369
- end
370
- private_class_method :junk_title_ratio_failure?
371
-
372
- def short_title_failure?(metrics)
373
- metrics[:short_title_count].positive?
374
- end
375
- private_class_method :short_title_failure?
376
-
377
- def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
378
- return [nil, nil] if min_items_passed && !quality_failed
379
- unless min_items_passed
380
- return [FailureKind.coerce(:min_items),
381
- "Extracted #{item_count} items (minimum required: #{min_items})"]
382
- end
383
-
384
- [FailureKind.coerce(:quality), quality_failure_message(quality_report)]
385
- end
386
- private_class_method :outcome_failure
387
-
388
- def quality_failure_message(quality_report)
389
- reasons = quality_failure_reasons(quality_report.metrics)
390
- "Feed quality check failed (#{reasons.join(', ')})"
391
- end
392
- private_class_method :quality_failure_message
393
-
394
- def quality_failure_reasons(metrics)
395
- reasons = []
396
- reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
397
- reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
398
- reasons << 'short_titles' if short_title_failure?(metrics)
399
- reasons
400
- end
401
- private_class_method :quality_failure_reasons
402
-
403
- def log_strict_quality_failure(failure_kind, item_count)
404
- Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
405
- end
406
- private_class_method :log_strict_quality_failure
407
-
408
- def probe_native_feed_url(channel_url, raw_config)
362
+ def probe_native_feed_url(channel_url, config)
409
363
  return nil if channel_url.to_s.empty?
410
364
 
411
- config = Config.from_hash(raw_config)
412
365
  resources = FeedPipeline::RuntimePolicy.resources_for(config)
413
- strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(raw_config[:strategy])
366
+ strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(config.strategy)
414
367
  session = RequestSession.build(
415
368
  config:, strategy:, budget: resources.budget, policy: resources.policy
416
369
  )
@@ -420,7 +373,7 @@ module Html2rss
420
373
  end
421
374
  private_class_method :probe_native_feed_url
422
375
 
423
- def validation_failure_result(errors, raw_config) # rubocop:disable Metrics/MethodLength
376
+ def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
424
377
  Result.new(
425
378
  success: false,
426
379
  item_count: 0,
@@ -429,9 +382,9 @@ module Html2rss
429
382
  channel_url: raw_config.dig(:channel, :url),
430
383
  strategy_used: raw_config[:strategy],
431
384
  duration_seconds: 0.0,
432
- validation_errors: errors,
385
+ validation_issues: report.issues,
433
386
  error_message: 'Configuration schema validation failed',
434
- failure_kind: FailureKind.coerce(:schema),
387
+ failure_kind: FailureKind[:schema],
435
388
  rss: nil,
436
389
  quality_report: nil
437
390
  )
@@ -447,9 +400,9 @@ module Html2rss
447
400
  channel_url: raw_config.dig(:channel, :url),
448
401
  strategy_used: raw_config[:strategy],
449
402
  duration_seconds: duration.round(3),
450
- validation_errors: nil,
403
+ validation_issues: nil,
451
404
  error_message: "#{error.class}: #{error.message}",
452
- failure_kind: FailureKind.coerce(:execution),
405
+ failure_kind: FailureKind[:execution],
453
406
  rss: nil,
454
407
  quality_report: nil
455
408
  )
data/lib/html2rss/url.rb CHANGED
@@ -42,14 +42,21 @@ module Html2rss
42
42
  url = Addressable::URI.parse(relative_url.to_s.strip)
43
43
  return new(url) if url.absolute?
44
44
 
45
- base_uri = Addressable::URI.parse(base_url.to_s)
46
- base_uri.path = '/' if base_uri.path.empty?
47
-
48
- new(base_uri.join(url).normalize)
45
+ base_url.is_a?(self) ? base_url.join_uri(url) : join_parsed_base(base_url, url)
49
46
  rescue Addressable::URI::InvalidURIError
50
47
  raise ArgumentError, 'URL could not be parsed'
51
48
  end
52
49
 
50
+ # @param base_url [String]
51
+ # @param url [Addressable::URI]
52
+ # @return [Url]
53
+ def self.join_parsed_base(base_url, url)
54
+ uri = Addressable::URI.parse(base_url.to_s)
55
+ uri.path = '/' if uri.path.empty?
56
+ new(uri.join(url).normalize)
57
+ end
58
+ private_class_method :join_parsed_base
59
+
53
60
  ##
54
61
  # Creates a URL by sanitizing a raw URL string.
55
62
  # Removes spaces and extracts the first valid URL from the string.
@@ -305,6 +312,20 @@ module Html2rss
305
312
  # @return [String] the debug representation
306
313
  def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
307
314
 
315
+ ##
316
+ # Resolves +relative_uri+ against this URL's already-parsed Addressable URI.
317
+ #
318
+ # @param relative_uri [Addressable::URI]
319
+ # @return [Url]
320
+ def join_uri(relative_uri)
321
+ base = @uri
322
+ if base.path.empty?
323
+ base = base.dup
324
+ base.path = '/'
325
+ end
326
+ self.class.new(base.join(relative_uri).normalize)
327
+ end
328
+
308
329
  protected
309
330
 
310
331
  # Scheme, authority, slash-stripped path, and query (fragment omitted).
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Html2rss
4
4
  # Current application version.
5
- VERSION = '0.30.0'
5
+ VERSION = '0.31.0'
6
6
  public_constant :VERSION
7
7
  end
data/lib/html2rss.rb CHANGED
@@ -11,20 +11,30 @@ loader.inflector.inflect(
11
11
  loader.setup
12
12
 
13
13
  require 'logger'
14
- require 'forwardable'
15
14
  require 'html2rss/defaults'
16
15
 
17
16
  ##
18
17
  # The Html2rss namespace.
19
18
  module Html2rss # rubocop:disable Metrics/ModuleLength
20
19
  ##
21
- # The logger instance.
20
+ # Forwards debug/info/warn/error to {Html2rss.logger}.
22
21
  module Log
23
22
  class << self
24
- extend Forwardable
23
+ ##
24
+ # @return [true]
25
+ def debug(...) = Html2rss.logger.debug(...)
25
26
 
26
- def_delegator 'Html2rss', :logger
27
- def_delegators :logger, :debug, :info, :warn, :error, :fatal, :unknown, :level, :level=, :formatter, :formatter=
27
+ ##
28
+ # @return [true]
29
+ def info(...) = Html2rss.logger.info(...)
30
+
31
+ ##
32
+ # @return [true]
33
+ def warn(...) = Html2rss.logger.warn(...)
34
+
35
+ ##
36
+ # @return [true]
37
+ def error(...) = Html2rss.logger.error(...)
28
38
  end
29
39
  end
30
40
 
@@ -79,7 +89,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
79
89
  # @param config_input [Hash, String]
80
90
  # @param feed_name [String, nil]
81
91
  # @param params [Hash]
82
- # @return [Dry::Validation::Result, Html2rss::Config::ValidationResult]
92
+ # @return [Html2rss::Config::ValidationReport]
83
93
  def self.validate(config_input, feed_name = nil, params: {})
84
94
  _raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
85
95
  validation
@@ -179,7 +189,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
179
189
  # @param pretty [Boolean] whether to pretty-print JSON
180
190
  # @return [String]
181
191
  def self.schema_json(pretty: true)
182
- Config.json_schema_json(pretty:)
192
+ Config::Schema.json_schema_json(pretty:)
183
193
  end
184
194
 
185
195
  ##
@@ -319,26 +329,24 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
319
329
  end
320
330
 
321
331
  ##
322
- # @return [Object] the logger
332
+ # @return [Logger] the logger
323
333
  def logger
324
334
  defaults.logger
325
335
  end
326
336
 
327
337
  ##
328
- # @param logger [Object] the new logger
338
+ # @param logger [Logger] the new logger
329
339
  def logger=(logger)
330
340
  configure { |config| config.logger = logger }
331
341
  end
332
342
 
333
- private
334
-
335
343
  ##
336
344
  # Resets the global defaults (mainly for testing).
337
345
  #
338
346
  # @return [void]
339
347
  def reset_defaults!
340
348
  @defaults = nil
341
- logger.level = defaults.log_level if logger.respond_to?(:level=)
349
+ logger.level = defaults.log_level
342
350
  end
343
351
  end
344
352
 
@@ -362,7 +370,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
362
370
  end
363
371
  end
364
372
 
365
- logger.level = defaults.log_level if logger.respond_to?(:level=)
373
+ logger.level = defaults.log_level
366
374
  end
367
375
 
368
376
  loader.eager_load
@@ -7,10 +7,10 @@ require_relative '../html2rss'
7
7
  namespace :config do
8
8
  desc 'Generate config JSON schema'
9
9
  task :schema do
10
- destination = Html2rss::Config.schema_path
10
+ destination = Html2rss::Config::Schema.path
11
11
 
12
12
  FileUtils.mkdir_p(File.dirname(destination))
13
- File.write(destination, "#{Html2rss::Config.json_schema_json}\n")
13
+ File.write(destination, "#{Html2rss::Config::Schema.json_schema_json}\n")
14
14
 
15
15
  puts "Generated config schema at #{destination}"
16
16
  end
@@ -434,7 +434,8 @@
434
434
  "properties": {
435
435
  "selector": {
436
436
  "type": "string",
437
- "minLength": 1
437
+ "minLength": 1,
438
+ "default": "a[href]"
438
439
  },
439
440
  "order": {
440
441
  "enum": [
@@ -507,9 +508,7 @@
507
508
  ]
508
509
  }
509
510
  },
510
- "required": [
511
- "selector"
512
- ],
511
+ "required": [],
513
512
  "description": "Defines the items selector and list-card enhance settings."
514
513
  },
515
514
  "enclosure": {
@@ -848,7 +847,7 @@
848
847
  "gsub": {
849
848
  "type": "object",
850
849
  "title": "gsub",
851
- "description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string).",
850
+ "description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string). Patterns over 256 characters as written, or with nested quantifiers, are rejected.",
852
851
  "examples": [
853
852
  {
854
853
  "name": "gsub",
@@ -862,10 +861,14 @@
862
861
  "const": "gsub"
863
862
  },
864
863
  "pattern": {
865
- "type": "string"
864
+ "type": "string",
865
+ "maxLength": 256
866
866
  },
867
867
  "replacement": {
868
- "type": "string"
868
+ "type": [
869
+ "string",
870
+ "object"
871
+ ]
869
872
  }
870
873
  },
871
874
  "required": [
@@ -1024,7 +1027,7 @@
1024
1027
  "type": "string",
1025
1028
  "const": "href",
1026
1029
  "title": "href",
1027
- "description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the channel URL).",
1030
+ "description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the page base URL).",
1028
1031
  "examples": [
1029
1032
  "href"
1030
1033
  ]