html2rss 0.29.1 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -8
  3. data/html2rss.gemspec +1 -3
  4. data/lib/html2rss/article/deduplicator.rb +2 -4
  5. data/lib/html2rss/article.rb +16 -39
  6. data/lib/html2rss/auto_source/README.md +1 -1
  7. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  8. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  10. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  11. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  13. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  14. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  15. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  16. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  17. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  19. data/lib/html2rss/auto_source/scraper.rb +4 -14
  20. data/lib/html2rss/auto_source/segment.rb +7 -4
  21. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  22. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  23. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  24. data/lib/html2rss/auto_source.rb +2 -2
  25. data/lib/html2rss/batch.rb +6 -4
  26. data/lib/html2rss/capture/README.md +7 -5
  27. data/lib/html2rss/capture.rb +68 -36
  28. data/lib/html2rss/channel.rb +3 -3
  29. data/lib/html2rss/cli/probe_view.rb +5 -3
  30. data/lib/html2rss/cli/render.rb +6 -5
  31. data/lib/html2rss/cli/validate.rb +28 -27
  32. data/lib/html2rss/cli.rb +1 -1
  33. data/lib/html2rss/config/issue_mapper.rb +108 -0
  34. data/lib/html2rss/config/request_headers.rb +6 -4
  35. data/lib/html2rss/config/schema.rb +16 -1
  36. data/lib/html2rss/config/selectors_validator.rb +92 -70
  37. data/lib/html2rss/config/validation_issue.rb +60 -0
  38. data/lib/html2rss/config/validation_report.rb +55 -0
  39. data/lib/html2rss/config/validator.rb +10 -6
  40. data/lib/html2rss/config.rb +35 -101
  41. data/lib/html2rss/defaults.rb +42 -30
  42. data/lib/html2rss/doctor/botasaurus.rb +19 -9
  43. data/lib/html2rss/doctor.rb +8 -0
  44. data/lib/html2rss/error.rb +1 -1
  45. data/lib/html2rss/feed_builder/rss.rb +9 -14
  46. data/lib/html2rss/feed_pipeline/README.md +8 -8
  47. data/lib/html2rss/feed_pipeline/auto_fallback.rb +1 -1
  48. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  49. data/lib/html2rss/feed_pipeline/strategy_plan.rb +7 -3
  50. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  51. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  52. data/lib/html2rss/feed_resolution/options.rb +3 -1
  53. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  54. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  55. data/lib/html2rss/html/article_extractor.rb +2 -4
  56. data/lib/html2rss/html/feed_link.rb +3 -1
  57. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  58. data/lib/html2rss/html/navigator.rb +10 -10
  59. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  60. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  61. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  62. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  63. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  64. data/lib/html2rss/mcp/README.md +5 -3
  65. data/lib/html2rss/mcp/config_argument.rb +3 -1
  66. data/lib/html2rss/mcp/contract.rb +15 -25
  67. data/lib/html2rss/mcp/outcome/playbook.rb +7 -7
  68. data/lib/html2rss/mcp/outcome.rb +36 -43
  69. data/lib/html2rss/mcp/server/tools.rb +112 -132
  70. data/lib/html2rss/mcp/server.rb +27 -19
  71. data/lib/html2rss/page_recon/README.md +5 -5
  72. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  73. data/lib/html2rss/page_recon.rb +9 -5
  74. data/lib/html2rss/recon.rb +20 -31
  75. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  76. data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
  77. data/lib/html2rss/request_service/compressed_body.rb +17 -9
  78. data/lib/html2rss/request_service/httpx_strategy.rb +239 -0
  79. data/lib/html2rss/request_service/network_guard.rb +7 -5
  80. data/lib/html2rss/request_service/response.rb +1 -2
  81. data/lib/html2rss/request_service/response_guard.rb +0 -16
  82. data/lib/html2rss/request_service/strategy.rb +22 -11
  83. data/lib/html2rss/request_service.rb +46 -85
  84. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  85. data/lib/html2rss/scoring/engine.rb +8 -8
  86. data/lib/html2rss/scoring/observation.rb +5 -3
  87. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  88. data/lib/html2rss/scrape_target.rb +3 -1
  89. data/lib/html2rss/selector_candidates.rb +234 -0
  90. data/lib/html2rss/selectors/README.md +64 -0
  91. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  92. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  93. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  94. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  95. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  96. data/lib/html2rss/selectors/extractors.rb +33 -20
  97. data/lib/html2rss/selectors/item_env.rb +31 -0
  98. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  99. data/lib/html2rss/selectors/option_spec.rb +140 -0
  100. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  101. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  102. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  103. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  104. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  105. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  106. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  107. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  108. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  109. data/lib/html2rss/selectors/post_processors.rb +6 -5
  110. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  111. data/lib/html2rss/selectors/step_env.rb +24 -0
  112. data/lib/html2rss/selectors.rb +231 -142
  113. data/lib/html2rss/sst/attrs.rb +5 -3
  114. data/lib/html2rss/sst/document.rb +3 -1
  115. data/lib/html2rss/sst/index.rb +3 -1
  116. data/lib/html2rss/sst/node.rb +29 -10
  117. data/lib/html2rss/sst/normalizer.rb +12 -16
  118. data/lib/html2rss/sst/tags.rb +7 -7
  119. data/lib/html2rss/status.rb +13 -8
  120. data/lib/html2rss/surface_category.rb +14 -32
  121. data/lib/html2rss/syndication/parser.rb +2 -2
  122. data/lib/html2rss/test/enhance_audit.rb +1 -3
  123. data/lib/html2rss/test/policy.rb +106 -0
  124. data/lib/html2rss/test.rb +126 -172
  125. data/lib/html2rss/url.rb +25 -4
  126. data/lib/html2rss/version.rb +1 -1
  127. data/lib/html2rss.rb +27 -19
  128. data/lib/tasks/config_schema.rake +2 -2
  129. data/schema/html2rss-config.schema.json +11 -8
  130. metadata +19 -44
  131. data/lib/html2rss/request_service/faraday_strategy.rb +0 -233
  132. data/lib/html2rss/selectors/item_scope.rb +0 -30
@@ -13,53 +13,8 @@ module Html2rss
13
13
  class Config # rubocop:disable Metrics/ClassLength
14
14
  # Raised when a configuration hash fails runtime validation.
15
15
  class InvalidConfig < Html2rss::Error; end
16
- # Sentinel to differentiate omitted params from explicit `nil`.
17
- UNSET = Object.new.freeze
18
-
19
- ValidationResult = Data.define(:success, :errors_hash)
20
-
21
- ##
22
- # Duck-compatible with +Dry::Validation::Result+ for resolve-time outcomes
23
- # (+#success?+, +#errors#to_h+). Used for parse failures so callers never see a Struct snowflake.
24
- class ValidationResult
25
- ##
26
- # @param message [String]
27
- # @return [Html2rss::Config::ValidationResult]
28
- def self.parse_failure(message)
29
- new(success: false, errors_hash: { parse: [message] })
30
- end
31
-
32
- ##
33
- # @return [Boolean]
34
- def success? = success
35
-
36
- ##
37
- # @return [self]
38
- def errors = self
39
-
40
- ##
41
- # @return [Hash]
42
- def to_h = errors_hash
43
- end
44
16
 
45
17
  class << self
46
- ##
47
- # Returns the exported JSON Schema for html2rss configuration.
48
- #
49
- # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
50
- def json_schema
51
- Schema.json_schema
52
- end
53
-
54
- ##
55
- # Returns the exported JSON Schema as JSON.
56
- #
57
- # @param pretty [Boolean] whether to pretty-print the JSON output
58
- # @return [String] serialized JSON Schema
59
- def json_schema_json(pretty: true)
60
- pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
61
- end
62
-
63
18
  ##
64
19
  # Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it.
65
20
  # The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.
@@ -67,13 +22,14 @@ module Html2rss
67
22
  # @param config_input [Hash, String] config hash, YAML string, or file path
68
23
  # @param feed_name [String, nil] optional feed name for multi-feed files
69
24
  # @param params [Hash] dynamic feed params
70
- # @return [Array(Hash, Dry::Validation::Result, Html2rss::Config::ValidationResult)]
25
+ # @return [Array(Hash, Html2rss::Config::ValidationReport)]
71
26
  def resolve_and_validate(config_input, feed_name: nil, params: {})
72
- param_arg = params.empty? ? UNSET : params
73
- working = HashUtil.deep_dup(resolve_raw_hash(config_input, feed_name))
74
- [working, validate(working, params: param_arg)]
75
- rescue StandardError => error
76
- [{}, ValidationResult.parse_failure(error.message)]
27
+ raw = resolve_raw_hash(config_input, feed_name)
28
+ rescue Psych::Exception, ArgumentError => error
29
+ [{}, IssueMapper.parse_failure(error.message)]
30
+ else
31
+ working = HashUtil.deep_dup(raw)
32
+ [working, validate(working, params:)]
77
33
  end
78
34
 
79
35
  ##
@@ -81,24 +37,15 @@ module Html2rss
81
37
  #
82
38
  # @param config [Hash{Symbol => Object}] the configuration hash
83
39
  # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
84
- # @return [Dry::Validation::Result] validation result after defaults are applied
85
- def validate(config, params: UNSET)
40
+ # @return [Html2rss::Config::ValidationReport]
41
+ def validate(config, params: {})
86
42
  prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
87
-
88
- Validator.new.call(prepared_config)
43
+ IssueMapper.from(Validator.new.call(prepared_config), values: prepared_config)
89
44
  rescue DynamicParams::ParamsMissing => error
90
45
  prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
91
46
  prepared_config[:dynamic_params_error] = error.message
92
47
 
93
- Validator.new.call(prepared_config)
94
- end
95
-
96
- ##
97
- # Returns the packaged JSON Schema file path.
98
- #
99
- # @return [String] absolute path to the packaged JSON Schema file
100
- def schema_path
101
- Schema.path
48
+ IssueMapper.from(Validator.new.call(prepared_config), values: prepared_config)
102
49
  end
103
50
 
104
51
  ##
@@ -167,7 +114,7 @@ module Html2rss
167
114
  # @param config [Hash{Symbol => Object}] the configuration hash.
168
115
  # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
169
116
  # @return [Html2rss::Config] the configuration object.
170
- def from_hash(config, params: UNSET)
117
+ def from_hash(config, params: {})
171
118
  new(resolve_effective_config(config, params:))
172
119
  end
173
120
 
@@ -214,6 +161,20 @@ module Html2rss
214
161
  Html2rss.defaults.default_strategy || :auto
215
162
  end
216
163
 
164
+ ##
165
+ # Normalizes raw config input with defaults.
166
+ #
167
+ # @param config [Hash{Symbol => Object}] raw config input
168
+ # @return [Hash{Symbol => Object}] config with defaults applied
169
+ def prepare_defaults(config)
170
+ config = HashUtil.deep_merge(default_config, config)
171
+ config = HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config) if config[:selectors]
172
+ if config[:auto_source]
173
+ config = HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
174
+ end
175
+ config
176
+ end
177
+
217
178
  private
218
179
 
219
180
  def default_request_config
@@ -227,7 +188,7 @@ module Html2rss
227
188
  def resolve_effective_config(config, params:)
228
189
  cfg = HashUtil.deep_symbolize_keys(config, context: 'config')
229
190
  p = parameter_defaults(cfg)
230
- p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.equal?(UNSET) || params.nil?
191
+ p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.nil?
231
192
 
232
193
  cfg[:headers] = DynamicParams.call(cfg[:headers], p) if cfg[:headers]
233
194
  cfg[:channel] = DynamicParams.call(cfg[:channel], p) if cfg[:channel]
@@ -243,7 +204,7 @@ module Html2rss
243
204
  end
244
205
 
245
206
  def prepare_for_validation(config)
246
- Config::Preparer.new.call(HashUtil.deep_dup(config))
207
+ prepare_defaults(HashUtil.deep_dup(config))
247
208
  end
248
209
 
249
210
  # @param config_input [Hash, String]
@@ -270,7 +231,7 @@ module Html2rss
270
231
  # @raise [InvalidConfig] if the configuration fails validation.
271
232
  def initialize(config)
272
233
  @request_controls = RequestControls.from_config(config)
273
- prepared_config = Preparer.new.call(config)
234
+ prepared_config = self.class.prepare_defaults(config.dup)
274
235
  validated_config = validated_config_for(prepared_config)
275
236
 
276
237
  @config = validated_config.freeze
@@ -328,42 +289,15 @@ module Html2rss
328
289
 
329
290
  attr_reader :config
330
291
 
331
- # Normalizes raw config input before validation.
332
- class Preparer
333
- ##
334
- # @param config [Hash{Symbol => Object}] raw config input
335
- # @return [Hash{Symbol => Object}] config with defaults applied
336
- def call(config)
337
- config = config.dup if config.frozen?
338
-
339
- config = apply_default_config(config)
340
- config = apply_default_selectors_config(config) if config[:selectors]
341
- config = apply_default_auto_source_config(config) if config[:auto_source]
342
-
343
- config
344
- end
345
-
346
- private
347
-
348
- def apply_default_config(config)
349
- HashUtil.deep_merge(Config.default_config, config)
350
- end
351
-
352
- def apply_default_selectors_config(config)
353
- HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
354
- end
355
-
356
- def apply_default_auto_source_config(config)
357
- HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
358
- end
359
- end
360
-
361
292
  def validated_config_for(config)
362
- validator = Validator.new.call(config)
293
+ dry = Validator.new.call(config)
363
294
 
364
- raise InvalidConfig, "Invalid configuration: #{validator.errors.to_h}" unless validator.success?
295
+ unless dry.success?
296
+ report = IssueMapper.from(dry, values: config)
297
+ raise InvalidConfig, "Invalid configuration: #{report}"
298
+ end
365
299
 
366
- normalized_headers(validator.to_h)
300
+ normalized_headers(dry.to_h)
367
301
  end
368
302
 
369
303
  def normalized_headers(validated_config)
@@ -4,13 +4,20 @@ module Html2rss
4
4
  ##
5
5
  # Global defaults for the Html2rss gem.
6
6
  class Defaults
7
- # The valid symbol log levels.
8
- VALID_LOG_LEVELS = %i[debug info warn error fatal unknown].freeze
9
-
10
- # @return [Object] the logger
7
+ # Named Logger severities accepted by {#log_level=}.
8
+ LOG_LEVELS = {
9
+ debug: Logger::DEBUG,
10
+ info: Logger::INFO,
11
+ warn: Logger::WARN,
12
+ error: Logger::ERROR,
13
+ fatal: Logger::FATAL,
14
+ unknown: Logger::UNKNOWN
15
+ }.freeze
16
+
17
+ # @return [Logger] the logger
11
18
  attr_reader :logger
12
19
 
13
- # @return [Proc, nil] the logger formatter
20
+ # @return [Proc, Logger::Formatter, nil] the logger formatter
14
21
  attr_reader :logger_formatter
15
22
 
16
23
  # @return [Symbol, Integer] the current log level
@@ -46,12 +53,15 @@ module Html2rss
46
53
  ##
47
54
  # Sets the logger.
48
55
  #
49
- # @param logger [Object]
50
- # @return [Object] the logger
56
+ # @param logger [Logger]
57
+ # @return [Logger] the logger
58
+ # @raise [ArgumentError] if logger is not a Logger
51
59
  def logger=(logger)
60
+ raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(::Logger)
61
+
52
62
  @logger = logger
53
- @logger.level = @log_level if @logger.respond_to?(:level=)
54
- @logger.formatter = @logger_formatter if @logger_formatter && @logger.respond_to?(:formatter=)
63
+ @logger.level = @log_level
64
+ @logger.formatter = @logger_formatter if @logger_formatter
55
65
  end
56
66
 
57
67
  ##
@@ -62,34 +72,39 @@ module Html2rss
62
72
  # @raise [ArgumentError] if the log level is invalid
63
73
  def log_level=(level)
64
74
  @log_level = normalize_log_level(level)
65
- @logger.level = @log_level if @logger.respond_to?(:level=)
75
+ @logger.level = @log_level
66
76
  end
67
77
 
68
78
  ##
69
79
  # Sets the logger formatter.
70
80
  #
71
- # @param formatter [Proc, #call, nil] the new logger formatter
72
- # @return [Proc, #call, nil] the new logger formatter
73
- # @raise [ArgumentError] if formatter does not respond to #call
81
+ # @param formatter [Proc, Logger::Formatter, nil] the new logger formatter
82
+ # @return [Proc, Logger::Formatter, nil] the new logger formatter
83
+ # @raise [ArgumentError] if formatter is not a Proc, Logger::Formatter, or nil
74
84
  def logger_formatter=(formatter)
75
- raise ArgumentError, 'formatter must respond to #call or be nil' if formatter && !formatter.respond_to?(:call)
76
-
77
- @logger_formatter = formatter
78
- @logger.formatter = @logger_formatter if @logger.respond_to?(:formatter=)
85
+ @logger_formatter = case formatter
86
+ when nil, Proc, Logger::Formatter
87
+ formatter
88
+ else
89
+ raise ArgumentError, 'formatter must be a Proc, Logger::Formatter, or nil'
90
+ end
91
+ @logger.formatter = @logger_formatter
79
92
  end
80
93
 
81
94
  ##
82
95
  # Sets the global request headers.
83
96
  #
84
- # @param headers [Hash, Proc, #call, nil] the HTTP request headers to globally apply
85
- # @return [Hash, Proc, #call, nil] the assigned headers
86
- # @raise [ArgumentError] if headers is not a Hash or callable
97
+ # @param headers [Hash, Proc, nil] the HTTP request headers to globally apply
98
+ # @return [Hash, Proc, nil] the assigned headers
99
+ # @raise [ArgumentError] if headers is not a Hash, Proc, or nil
87
100
  def headers=(headers)
88
- if headers && !headers.is_a?(Hash) && !headers.respond_to?(:call)
89
- raise ArgumentError, 'headers must be a Hash or respond to #call'
90
- end
91
-
92
- @headers = headers.is_a?(Hash) ? headers.dup.freeze : headers
101
+ @headers = case headers
102
+ when nil then nil
103
+ when Hash then headers.dup.freeze
104
+ when Proc then headers
105
+ else
106
+ raise ArgumentError, 'headers must be a Hash or Proc'
107
+ end
93
108
  end
94
109
 
95
110
  ##
@@ -164,14 +179,11 @@ module Html2rss
164
179
 
165
180
  def normalize_log_level(level)
166
181
  if level.is_a?(Integer)
167
- raise ArgumentError, "invalid log level: #{level}" unless level.between?(0, 5)
182
+ raise ArgumentError, "invalid log level: #{level}" unless LOG_LEVELS.value?(level)
168
183
 
169
184
  level
170
185
  else
171
- sym = level.to_s.downcase.to_sym
172
- raise ArgumentError, "invalid log level: #{level}" unless VALID_LOG_LEVELS.include?(sym)
173
-
174
- Logger.const_get(sym.upcase)
186
+ LOG_LEVELS.fetch(level.to_s.downcase.to_sym) { raise ArgumentError, "invalid log level: #{level}" }
175
187
  end
176
188
  end
177
189
  end
@@ -1,18 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'faraday'
3
+ require 'httpx'
4
4
  require 'json'
5
5
 
6
6
  module Html2rss
7
- ##
8
- # CLI runtime health checks for optional scrape dependencies.
9
7
  module Doctor
10
8
  ##
11
9
  # Preflight checks for Botasaurus scrape-api connectivity.
12
10
  module Botasaurus
11
+ Check = Data.define(:name, :ok, :detail)
12
+
13
13
  ##
14
14
  # One named doctor check outcome.
15
- Check = Data.define(:name, :ok, :detail) do
15
+ class Check
16
16
  ##
17
17
  # @return [Hash{Symbol => Object}]
18
18
  def to_h
@@ -20,9 +20,11 @@ module Html2rss
20
20
  end
21
21
  end
22
22
 
23
+ Result = Data.define(:ok, :checks, :message)
24
+
23
25
  ##
24
26
  # Aggregate doctor command outcome.
25
- Result = Data.define(:ok, :checks, :message) do
27
+ class Result
26
28
  ##
27
29
  # @return [Hash{Symbol => Object}]
28
30
  def to_h
@@ -68,15 +70,23 @@ module Html2rss
68
70
  # @param base_url [String]
69
71
  # @return [Hash{Symbol => Object}]
70
72
  def health_check(base_url)
71
- uri = Url.for_channel(base_url)
72
- client = Faraday.new(url: uri.to_s.chomp('/'), request: { timeout: 5 })
73
- response = client.get('/health')
74
- body = JSON.parse(response.body)
73
+ endpoint = "#{Url.for_channel(base_url).to_s.chomp('/')}/health"
74
+ response = health_session.get(endpoint)
75
+ raise response.error if response.is_a?(HTTPX::ErrorResponse)
76
+
77
+ body = JSON.parse(response.body.to_s)
75
78
  { ok: response.status == 200, status: response.status, version: body['version'] }
76
79
  rescue StandardError => error
77
80
  { ok: false, error: "#{error.class}: #{error.message}" }
78
81
  end
79
82
 
83
+ def health_session
84
+ HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
85
+ resolver_class: :system)
86
+ end
87
+ module_function :health_session
88
+ private_class_method :health_session
89
+
80
90
  ##
81
91
  # @param url [String]
82
92
  # @return [Hash{Symbol => Object}]
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ ##
5
+ # CLI runtime health checks for optional scrape dependencies.
6
+ module Doctor
7
+ end
8
+ end
@@ -7,7 +7,7 @@ module Html2rss
7
7
  # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
8
8
  class NoFeedItemsExtracted < Error
9
9
  # Categories that append shared surface guidance to the empty-feed message.
10
- SURFACE_HINT_CATEGORIES = %i[app_shell blocked_surface high_entropy_surface].freeze
10
+ SURFACE_HINT_CATEGORIES = Set[:app_shell, :blocked_surface, :high_entropy_surface].freeze
11
11
 
12
12
  ##
13
13
  # @param attempts [Array<Hash{Symbol => Object}>] tier attempt diagnostics
@@ -34,16 +34,10 @@ module Html2rss
34
34
  private
35
35
 
36
36
  def add_item_string_values(article, item_maker)
37
- {
38
- title: article.title,
39
- description: ItemPresentation.description_for(article),
40
- author: article.author
41
- }.each do |attr, value|
42
- next if value.nil? || value.empty?
43
-
44
- item_maker.public_send(:"#{attr}=", value)
45
- end
46
-
37
+ item_maker.title = article.title
38
+ item_maker.author = article.author
39
+ description = ItemPresentation.description_for(article)
40
+ item_maker.description = description if description && !description.empty?
47
41
  item_maker.link = article.url.to_s if article.url
48
42
  item_maker.pubDate = article.published_at&.rfc2822
49
43
  end
@@ -96,12 +90,13 @@ module Html2rss
96
90
  @stylesheets.map { |style| Stylesheet.new(**style) }
97
91
  end
98
92
 
99
- # rubocop:disable-next Metrics/AbcSize
93
+ # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
100
94
  def make_channel(maker)
101
95
  channel_maker = maker.channel
102
- %i[language title description ttl].each do |key|
103
- channel_maker.public_send(:"#{key}=", channel.public_send(key))
104
- end
96
+ channel_maker.language = channel.language
97
+ channel_maker.title = channel.title
98
+ channel_maker.description = channel.description
99
+ channel_maker.ttl = channel.ttl
105
100
 
106
101
  channel_maker.managingEditor = channel.author if channel.author
107
102
  channel_maker.author = channel.author if channel.author
@@ -2,25 +2,25 @@
2
2
 
3
3
  `:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape` / `capture`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
4
4
 
5
- Use `:auto` when you want Faraday first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`faraday`, `botasaurus`, `local_file`) when you need a single transport.
5
+ Use `:auto` when you want `default` (HTTPX) first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`default`, `botasaurus`, `local_file`) when you need a single transport.
6
6
 
7
7
  ## Chain
8
8
 
9
9
  `AutoFallback::CHAIN` is:
10
10
 
11
- 1. **Faraday** — plain HTTP (faster, cheaper).
12
- 2. **Botasaurus** — attempted when Faraday raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
11
+ 1. **Default (HTTPX)** — plain HTTP (faster, cheaper).
12
+ 2. **Botasaurus** — attempted when default raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
13
13
 
14
- Before escalating Faraday → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
14
+ Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
15
15
 
16
- There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without Faraday first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
16
+ There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without default first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
17
17
 
18
18
  ## Surfaces
19
19
 
20
20
  | Surface | `:auto` behavior |
21
21
  | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
22
- | Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`) |
23
- | MCP `inspect` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
22
+ | Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`default` → `botasaurus`) |
23
+ | MCP `inspect` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to default (pin `botasaurus` when you need browser rendering) |
24
24
 
25
25
  ## Fallback vs abort
26
26
 
@@ -28,7 +28,7 @@ These typically hop to the next chain member (among other `StandardError`s `Auto
28
28
 
29
29
  - `Html2rss::RequestService::BlockedSurfaceDetected`
30
30
  - `Html2rss::RequestService::RequestTimedOut`
31
- - Faraday connection / timeout errors
31
+ - HTTPX connection / timeout errors
32
32
  - Empty extraction results when a later chain member may succeed
33
33
 
34
34
  These abort immediately (`AutoFallback::NON_FALLBACK_ERRORS`): unknown strategy, invalid URL, unsupported scheme, budget exceeded, private network denied, cross-origin follow-up denied, response too large.
@@ -8,7 +8,7 @@ module Html2rss
8
8
  # Hosted by the pipeline instance: session + extract call back into FeedPipeline.
9
9
  class AutoFallback # rubocop:disable Metrics/ClassLength -- attempt state + logging stay co-located
10
10
  # Ordered list of concrete request strategies attempted by the :auto plan.
11
- CHAIN = %i[faraday botasaurus].freeze
11
+ CHAIN = %i[default botasaurus].freeze
12
12
 
13
13
  # Deterministic HTTP statuses that indicate permanent failure rather than anti-bot challenges.
14
14
  DETERMINISTIC_HTTP_STATUSES = Set[400, 404, 410, 422, 451].freeze
@@ -45,6 +45,20 @@ module Html2rss
45
45
  # @return [Html2rss::RequestService::Budget] shared budget for the feed build
46
46
  def self.budget_for(config) = resources_for(config).budget
47
47
 
48
+ ##
49
+ # Reserve enough HTTP request slots for the initial request plus predictable
50
+ # follow-ups.
51
+ #
52
+ # @param config [Html2rss::Config] validated feed config
53
+ # @return [Integer] number of reserved baseline request slots
54
+ def self.baseline_request_budget_for(config)
55
+ 1 +
56
+ RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
57
+ AutoSource.request_slots_for(config.auto_source) +
58
+ FeedResolution.request_slots_for(config.auto_source) +
59
+ StrategyPlan.resolve(config.strategy).request_slots
60
+ end
61
+
48
62
  class << self
49
63
  private
50
64
 
@@ -53,16 +67,6 @@ module Html2rss
53
67
 
54
68
  [baseline_request_budget_for(config), config.max_requests].max
55
69
  end
56
-
57
- # Reserve enough HTTP request slots for the initial request plus predictable
58
- # follow-ups.
59
- def baseline_request_budget_for(config)
60
- 1 +
61
- RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
62
- AutoSource.request_slots_for(config.auto_source) +
63
- FeedResolution.request_slots_for(config.auto_source) +
64
- StrategyPlan.resolve(config.strategy).request_slots
65
- end
66
70
  end
67
71
  end
68
72
  end
@@ -8,16 +8,20 @@ module Html2rss
8
8
  # {RequestService} executes concrete adapters only. +:auto+ is resolved here before
9
9
  # any session or adapter call — plan vs transport locality.
10
10
  class StrategyPlan
11
+ Auto = Data.define
12
+
11
13
  # Feed-level auto plan: try {AutoFallback::CHAIN} until items are extracted.
12
- Auto = Data.define do
14
+ class Auto
13
15
  # @return [Integer]
14
16
  def request_slots
15
17
  [AutoFallback::CHAIN.size - 1, 0].max
16
18
  end
17
19
  end
18
20
 
21
+ Concrete = Data.define(:strategy)
22
+
19
23
  # Concrete transport strategy name for {RequestService}.
20
- Concrete = Data.define(:strategy) do
24
+ class Concrete
21
25
  # @return [Integer]
22
26
  def request_slots = 0
23
27
  end
@@ -42,7 +46,7 @@ module Html2rss
42
46
  end
43
47
 
44
48
  ##
45
- # Cheap single-request diagnostic default: +:auto+ → +:faraday+.
49
+ # Cheap single-request diagnostic default: +:auto+ → +:default+.
46
50
  # Does not run {AutoFallback}. Prefer {resolve} for scrape/capture feeds.
47
51
  #
48
52
  # @param name [Symbol, String, nil] plan name (+nil+ → +:auto+)
@@ -99,11 +99,8 @@ module Html2rss
99
99
 
100
100
  def segment_urls_for(sst, strategy, link_resolver)
101
101
  segmenter = AutoSource::Segmenter.new(sst, base_url: entry_url, strategy:, link_resolver:)
102
- primary = AutoSource::Segmenter::PrimaryLink.new(segmenter)
103
-
104
102
  segmenter.call.filter_map do |segment|
105
- link = segment.primary_link || primary.select(segment.root_node)
106
- absolute_same_origin(link&.attrs&.href)
103
+ absolute_same_origin(segment.primary_link&.attrs&.href)
107
104
  end.uniq
108
105
  end
109
106
 
@@ -2,9 +2,11 @@
2
2
 
3
3
  module Html2rss
4
4
  module FeedResolution
5
+ Diag = Data.define(:applied, :probe_count, :reason, :winner_score)
6
+
5
7
  ##
6
8
  # Wire-safe diagnostics for one entry-resolution attempt (Status / Marshal edge).
7
- Diag = Data.define(:applied, :probe_count, :reason, :winner_score) do
9
+ class Diag
8
10
  ##
9
11
  # @param result [Html2rss::FeedResolution::Result]
10
12
  # @return [Diag]
@@ -2,9 +2,11 @@
2
2
 
3
3
  module Html2rss
4
4
  module FeedResolution
5
+ Options = Data.define(:enabled, :max_probes)
6
+
5
7
  ##
6
8
  # Typed +auto_source.entry_resolution+ options (one expansion from config Hash).
7
- Options = Data.define(:enabled, :max_probes) do
9
+ class Options
8
10
  ##
9
11
  # @param auto_source [Hash, nil]
10
12
  # @return [Options]
@@ -13,13 +13,13 @@ module Html2rss
13
13
  ##
14
14
  # @param config [Html2rss::Config]
15
15
  # @param articles [Array<Html2rss::Article>] typed extract results (count derived)
16
- # @param surface_category [Html2rss::SurfaceCategory, Symbol, nil]
16
+ # @param surface_category [Html2rss::SurfaceCategory, Symbol]
17
17
  # @return [Boolean]
18
18
  def resolve?(config:, articles:, surface_category:)
19
19
  raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
20
20
  return false unless eligible_config?(config)
21
21
 
22
- category = SurfaceCategory.coerce(surface_category)
22
+ category = SurfaceCategory[surface_category]
23
23
  return false if category.blocked?
24
24
 
25
25
  articles.size < ARTICLE_FLOOR || category.weak? || native_feed_majority?(articles)
@@ -10,7 +10,7 @@ module Html2rss
10
10
  # @param time_zone [String] channel time zone for naive values
11
11
  # @return [DateTime, nil]
12
12
  def self.call(article_tag, leftover_lines: [], time_zone: 'UTC')
13
- datetimes = article_tag.respond_to?(:css) ? article_tag.css('[datetime]').map { |tag| tag['datetime'] } : []
13
+ datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
14
14
  ArticleRules::Date.earliest(datetimes, leftover_lines:, time_zone:)
15
15
  end
16
16
  end
@@ -124,8 +124,7 @@ module Html2rss
124
124
  end
125
125
 
126
126
  def leftover_exclude_nodes_for(node)
127
- times = node.respond_to?(:css) ? node.css('time') : []
128
- [heading, selected_anchor, kicker_node, *times].compact
127
+ [heading, selected_anchor, kicker_node, *node.css('time')].compact
129
128
  end
130
129
 
131
130
  def heading_or_anchor_item?
@@ -156,8 +155,7 @@ module Html2rss
156
155
  end
157
156
 
158
157
  def immediate_card_parent
159
- parent = article_tag.respond_to?(:parent) ? article_tag.parent : nil
160
- Navigator.parent_until_condition(parent, method(:usable_walk_parent?))
158
+ Navigator.parent_until_condition(article_tag.parent, ->(node) { usable_walk_parent?(node) })
161
159
  end
162
160
 
163
161
  def usable_walk_parent?(node)