html2rss 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +82 -25
  3. data/html2rss.gemspec +0 -1
  4. data/lib/html2rss/article.rb +1 -2
  5. data/lib/html2rss/auto_source/README.md +8 -8
  6. data/lib/html2rss/auto_source/cleanup.rb +92 -1
  7. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
  8. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
  9. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
  10. data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
  11. data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
  12. data/lib/html2rss/auto_source/scraper.rb +2 -3
  13. data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
  14. data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
  15. data/lib/html2rss/auto_source/segmenter.rb +1 -2
  16. data/lib/html2rss/auto_source.rb +25 -11
  17. data/lib/html2rss/batch.rb +169 -0
  18. data/lib/html2rss/capture/README.md +15 -6
  19. data/lib/html2rss/capture.rb +158 -15
  20. data/lib/html2rss/channel.rb +1 -2
  21. data/lib/html2rss/cli/probe_view.rb +66 -0
  22. data/lib/html2rss/cli/render.rb +124 -0
  23. data/lib/html2rss/cli/validate.rb +91 -0
  24. data/lib/html2rss/cli.rb +318 -193
  25. data/lib/html2rss/config/request_headers.rb +4 -14
  26. data/lib/html2rss/config/schema.rb +7 -5
  27. data/lib/html2rss/config/validator.rb +1 -0
  28. data/lib/html2rss/config.rb +59 -22
  29. data/lib/html2rss/doctor/botasaurus.rb +130 -0
  30. data/lib/html2rss/feed_builder/rss.rb +1 -2
  31. data/lib/html2rss/feed_pipeline/README.md +5 -5
  32. data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
  33. data/lib/html2rss/feed_pipeline.rb +27 -16
  34. data/lib/html2rss/feed_resolution/README.md +11 -11
  35. data/lib/html2rss/feed_resolution.rb +6 -12
  36. data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
  37. data/lib/html2rss/html/article_extractor.rb +4 -9
  38. data/lib/html2rss/html/feed_link.rb +4 -2
  39. data/lib/html2rss/html/navigator.rb +4 -39
  40. data/lib/html2rss/html/probe.rb +82 -0
  41. data/lib/html2rss/html/sst_article_extractor.rb +1 -2
  42. data/lib/html2rss/link_destination/noise_policy.rb +1 -2
  43. data/lib/html2rss/link_destination/path_classifier.rb +20 -19
  44. data/lib/html2rss/mcp/README.md +80 -0
  45. data/lib/html2rss/mcp/contract.rb +135 -13
  46. data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
  47. data/lib/html2rss/mcp/outcome.rb +112 -42
  48. data/lib/html2rss/mcp/runtime.rb +45 -0
  49. data/lib/html2rss/mcp/server/tools.rb +277 -0
  50. data/lib/html2rss/mcp/server.rb +91 -232
  51. data/lib/html2rss/mcp.rb +3 -0
  52. data/lib/html2rss/page_recon/README.md +69 -0
  53. data/lib/html2rss/page_recon/diagnostics.rb +211 -0
  54. data/lib/html2rss/page_recon.rb +56 -2
  55. data/lib/html2rss/recon.rb +252 -0
  56. data/lib/html2rss/request_service/blocked_surface.rb +1 -0
  57. data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
  58. data/lib/html2rss/request_service/policy.rb +1 -2
  59. data/lib/html2rss/request_service/response.rb +3 -4
  60. data/lib/html2rss/request_service/strategy.rb +1 -2
  61. data/lib/html2rss/request_service.rb +2 -0
  62. data/lib/html2rss/request_session.rb +1 -2
  63. data/lib/html2rss/scrape_target.rb +1 -1
  64. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
  65. data/lib/html2rss/selectors/post_processors.rb +0 -1
  66. data/lib/html2rss/selectors.rb +33 -10
  67. data/lib/html2rss/sst/attrs.rb +1 -2
  68. data/lib/html2rss/sst/normalizer.rb +49 -11
  69. data/lib/html2rss/status.rb +3 -6
  70. data/lib/html2rss/syndication/README.md +6 -6
  71. data/lib/html2rss/syndication/discovery.rb +2 -4
  72. data/lib/html2rss/test/enhance_audit.rb +192 -0
  73. data/lib/html2rss/test.rb +458 -0
  74. data/lib/html2rss/url.rb +1 -2
  75. data/lib/html2rss/version.rb +1 -1
  76. data/lib/html2rss.rb +157 -35
  77. data/schema/html2rss-config.schema.json +9 -29
  78. metadata +17 -18
  79. data/lib/html2rss/mcp/inspect.rb +0 -138
  80. data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
@@ -48,7 +48,7 @@ module Html2rss
48
48
  ##
49
49
  # Orchestrates schema assembly from runtime validator contracts plus
50
50
  # client-facing overlays.
51
- # rubocop:disable Metrics/ClassLength -- overlay assembly stays in one builder
51
+ # rubocop:disable-next Metrics/ClassLength -- overlay assembly stays in one builder
52
52
  class Builder
53
53
  class << self
54
54
  # @return [Hash{String => Object}] fully assembled JSON schema hash
@@ -110,9 +110,13 @@ module Html2rss
110
110
  registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
111
111
  end
112
112
 
113
- # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
113
+ # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
114
114
  def overlay
115
115
  items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
116
+ items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
117
+ description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
118
+ 'to fill missing fields from the card HTML.'
119
+ )
116
120
  items_schema[:properties][:pagination] = {
117
121
  description: 'Pagination configuration or maximum page count integer.',
118
122
  oneOf: [
@@ -166,7 +170,7 @@ module Html2rss
166
170
  description: 'Selectors used to extract article attributes.',
167
171
  properties: {
168
172
  items: items_schema.merge(
169
- description: 'Defines the items selector and optional enhancement settings.'
173
+ description: 'Defines the items selector and list-card enhance settings.'
170
174
  ),
171
175
  enclosure: enclosure_schema.merge(
172
176
  description: 'Describes enclosure extraction settings.'
@@ -183,7 +187,6 @@ module Html2rss
183
187
  }
184
188
  }
185
189
  end
186
- # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
187
190
 
188
191
  def reference_array(description)
189
192
  {
@@ -197,7 +200,6 @@ module Html2rss
197
200
  }
198
201
  end
199
202
  end
200
- # rubocop:enable Metrics/ClassLength
201
203
 
202
204
  ##
203
205
  # Wires extractor / post_process to thin oneOf $refs into the registry catalog.
@@ -19,6 +19,7 @@ module Html2rss
19
19
  DIRECTORY_TOPICS = %w[
20
20
  sports energy tech science news entertainment jobs finance
21
21
  security travel environment consumer civic product research
22
+ health culture education rights transport
22
23
  ].freeze
23
24
 
24
25
  # Contract for the top-level `channel` section.
@@ -16,6 +16,32 @@ module Html2rss
16
16
  # Sentinel to differentiate omitted params from explicit `nil`.
17
17
  UNSET = Object.new.freeze
18
18
 
19
+ ValidationResult = Data.define(:success, :errors_hash)
20
+
21
+ ##
22
+ # Duck-compatible with +Dry::Validation::Result+ for resolve-time outcomes
23
+ # (+#success?+, +#errors#to_h+). Used for parse failures so callers never see a Struct snowflake.
24
+ class ValidationResult
25
+ ##
26
+ # @param message [String]
27
+ # @return [Html2rss::Config::ValidationResult]
28
+ def self.parse_failure(message)
29
+ new(success: false, errors_hash: { parse: [message] })
30
+ end
31
+
32
+ ##
33
+ # @return [Boolean]
34
+ def success? = success
35
+
36
+ ##
37
+ # @return [self]
38
+ def errors = self
39
+
40
+ ##
41
+ # @return [Hash]
42
+ def to_h = errors_hash
43
+ end
44
+
19
45
  class << self
20
46
  ##
21
47
  # Returns the exported JSON Schema for html2rss configuration.
@@ -34,6 +60,22 @@ module Html2rss
34
60
  pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
35
61
  end
36
62
 
63
+ ##
64
+ # Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it.
65
+ # The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.
66
+ #
67
+ # @param config_input [Hash, String] config hash, YAML string, or file path
68
+ # @param feed_name [String, nil] optional feed name for multi-feed files
69
+ # @param params [Hash] dynamic feed params
70
+ # @return [Array(Hash, Dry::Validation::Result, Html2rss::Config::ValidationResult)]
71
+ def resolve_and_validate(config_input, feed_name: nil, params: {})
72
+ param_arg = params.empty? ? UNSET : params
73
+ working = HashUtil.deep_dup(resolve_raw_hash(config_input, feed_name))
74
+ [working, validate(working, params: param_arg)]
75
+ rescue StandardError => error
76
+ [{}, ValidationResult.parse_failure(error.message)]
77
+ end
78
+
37
79
  ##
38
80
  # Validates a configuration hash with the runtime validator.
39
81
  #
@@ -59,22 +101,10 @@ module Html2rss
59
101
  Schema.path
60
102
  end
61
103
 
62
- ##
63
- # Loads and validates a YAML configuration file.
64
- #
65
- # @param file [String] the YAML file to load
66
- # @param feed_name [String, nil] optional feed name for multi-feed files
67
- # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
68
- # @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
69
- # @return [Dry::Validation::Result] validation result after defaults are applied
70
- def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
71
- validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
72
- end
73
-
74
104
  ##
75
105
  # Serializes a configuration hash to string-key YAML.
76
106
  #
77
- # This is the single serializer for CLI capture and MCP +capture_config+.
107
+ # This is the single serializer for CLI capture and MCP +capture+.
78
108
  #
79
109
  # @param hash [Hash] configuration hash (symbol or string keys)
80
110
  # @return [String] YAML document without Ruby symbol-key prefixes
@@ -109,7 +139,7 @@ module Html2rss
109
139
  # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
110
140
  # @return [Hash{Symbol => Object}] the configuration hash.
111
141
  # @raise [ArgumentError] if the file doesn't exist or feed is not found.
112
- # rubocop:disable Metrics/MethodLength
142
+ # rubocop:disable-next Metrics/MethodLength
113
143
  def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
114
144
  raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
115
145
  raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
@@ -129,7 +159,6 @@ module Html2rss
129
159
 
130
160
  MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
131
161
  end
132
- # rubocop:enable Metrics/MethodLength
133
162
 
134
163
  ##
135
164
  # Processes the provided configuration hash, applying dynamic parameters if given,
@@ -216,6 +245,20 @@ module Html2rss
216
245
  def prepare_for_validation(config)
217
246
  Config::Preparer.new.call(HashUtil.deep_dup(config))
218
247
  end
248
+
249
+ # @param config_input [Hash, String]
250
+ # @param feed_name [String, nil]
251
+ # @return [Hash]
252
+ def resolve_raw_hash(config_input, feed_name)
253
+ if config_input.is_a?(Hash)
254
+ config_input
255
+ elsif File.file?(config_input.to_s)
256
+ load_yaml(config_input.to_s, feed_name)
257
+ else
258
+ from_yaml(config_input.to_s)
259
+ end
260
+ end
261
+ private :resolve_raw_hash
219
262
  end
220
263
 
221
264
  ##
@@ -270,11 +313,6 @@ module Html2rss
270
313
  # @return [String]
271
314
  def url = config.dig(:channel, :url)
272
315
 
273
- ##
274
- # @deprecated Use {ScrapeTarget} for entry vs effective URLs after resolution.
275
- # @return [String]
276
- def scrape_url = url
277
-
278
316
  # @return [String, nil] configured channel time zone
279
317
  def time_zone = config.dig(:channel, :time_zone)
280
318
 
@@ -331,8 +369,7 @@ module Html2rss
331
369
  def normalized_headers(validated_config)
332
370
  validated_config[:headers] = RequestHeaders.normalize(
333
371
  validated_config[:headers],
334
- channel_language: validated_config.dig(:channel, :language),
335
- url: validated_config.dig(:channel, :url)
372
+ channel_language: validated_config.dig(:channel, :language)
336
373
  )
337
374
  validated_config
338
375
  end
@@ -0,0 +1,130 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'faraday'
4
+ require 'json'
5
+
6
+ module Html2rss
7
+ ##
8
+ # CLI runtime health checks for optional scrape dependencies.
9
+ module Doctor
10
+ ##
11
+ # Preflight checks for Botasaurus scrape-api connectivity.
12
+ module Botasaurus
13
+ ##
14
+ # One named doctor check outcome.
15
+ Check = Data.define(:name, :ok, :detail) do
16
+ ##
17
+ # @return [Hash{Symbol => Object}]
18
+ def to_h
19
+ { name: name.to_s, ok:, detail: }
20
+ end
21
+ end
22
+
23
+ ##
24
+ # Aggregate doctor command outcome.
25
+ Result = Data.define(:ok, :checks, :message) do
26
+ ##
27
+ # @return [Hash{Symbol => Object}]
28
+ def to_h
29
+ { ok:, message:, checks: checks.map(&:to_h) }
30
+ end
31
+ end
32
+
33
+ module_function
34
+
35
+ ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
36
+ private_constant :ENV_VAR
37
+
38
+ ##
39
+ # @param sample_url [String, nil] optional URL for a sample scrape smoke test
40
+ # @return [Result]
41
+ def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
42
+ env = env_check
43
+ checks = [env_check_record(env)]
44
+ return missing_env_failure(checks) unless checks.first.ok
45
+
46
+ checks << health_check_record(env[:base_url])
47
+ return failure(checks, 'Botasaurus health check failed.') unless checks.last.ok
48
+
49
+ checks << sample_scrape_record(sample_url) if sample_url
50
+ return failure(checks, 'Sample scrape failed.') if sample_url && !checks.last.ok
51
+
52
+ Log.info('doctor botasaurus: preflight passed')
53
+ Result.new(ok: true, checks:, message: 'Botasaurus preflight passed.')
54
+ end
55
+
56
+ ##
57
+ # @return [Hash{Symbol => Object}]
58
+ def env_check
59
+ base = ENV.fetch(ENV_VAR, '').strip
60
+ {
61
+ configured: MCP::Runtime.botasaurus_configured?,
62
+ var: ENV_VAR,
63
+ base_url: base.empty? ? nil : base
64
+ }
65
+ end
66
+
67
+ ##
68
+ # @param base_url [String]
69
+ # @return [Hash{Symbol => Object}]
70
+ def health_check(base_url)
71
+ uri = Url.for_channel(base_url)
72
+ client = Faraday.new(url: uri.to_s.chomp('/'), request: { timeout: 5 })
73
+ response = client.get('/health')
74
+ body = JSON.parse(response.body)
75
+ { ok: response.status == 200, status: response.status, version: body['version'] }
76
+ rescue StandardError => error
77
+ { ok: false, error: "#{error.class}: #{error.message}" }
78
+ end
79
+
80
+ ##
81
+ # @param url [String]
82
+ # @return [Hash{Symbol => Object}]
83
+ def sample_scrape(url)
84
+ wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
85
+ {
86
+ ok: wire[:status].to_i.between?(200, 399),
87
+ status: wire[:status],
88
+ articles_count: wire[:articles_count]
89
+ }
90
+ rescue StandardError => error
91
+ { ok: false, error: "#{error.class}: #{error.message}" }
92
+ end
93
+
94
+ def env_check_record(env = env_check)
95
+ Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
96
+ end
97
+ module_function :env_check_record
98
+ private_class_method :env_check_record
99
+
100
+ def health_check_record(base_url)
101
+ return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
102
+
103
+ health = health_check(base_url)
104
+ Check.new(name: :health, ok: health[:ok], detail: health)
105
+ end
106
+ module_function :health_check_record
107
+ private_class_method :health_check_record
108
+
109
+ def sample_scrape_record(url)
110
+ sample = sample_scrape(url)
111
+ Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
112
+ end
113
+ module_function :sample_scrape_record
114
+ private_class_method :sample_scrape_record
115
+
116
+ def missing_env_failure(checks)
117
+ Log.info("doctor botasaurus: #{ENV_VAR} unset")
118
+ failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
119
+ end
120
+ module_function :missing_env_failure
121
+ private_class_method :missing_env_failure
122
+
123
+ def failure(checks, message)
124
+ Result.new(ok: false, checks:, message:)
125
+ end
126
+ module_function :failure
127
+ private_class_method :failure
128
+ end
129
+ end
130
+ end
@@ -96,7 +96,7 @@ module Html2rss
96
96
  @stylesheets.map { |style| Stylesheet.new(**style) }
97
97
  end
98
98
 
99
- # rubocop:disable Metrics/AbcSize
99
+ # rubocop:disable-next Metrics/AbcSize
100
100
  def make_channel(maker)
101
101
  channel_maker = maker.channel
102
102
  %i[language title description ttl].each do |key|
@@ -111,7 +111,6 @@ module Html2rss
111
111
 
112
112
  make_image(maker.image) if channel.image
113
113
  end
114
- # rubocop:enable Metrics/AbcSize
115
114
 
116
115
  def make_image(image_maker)
117
116
  image_maker.url = channel.image.to_s
@@ -1,6 +1,6 @@
1
1
  # FeedPipeline — `auto` request strategy
2
2
 
3
- `:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape_url` / `capture_config`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
3
+ `:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape` / `capture`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
4
4
 
5
5
  Use `:auto` when you want Faraday first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`faraday`, `botasaurus`, `local_file`) when you need a single transport.
6
6
 
@@ -17,10 +17,10 @@ There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want
17
17
 
18
18
  ## Surfaces
19
19
 
20
- | Surface | `:auto` behavior |
21
- |---------|------------------|
22
- | Gem / CLI feed build, MCP `scrape_url`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`) |
23
- | MCP `inspect_url` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
20
+ | Surface | `:auto` behavior |
21
+ | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
22
+ | Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`) |
23
+ | MCP `inspect` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
24
24
 
25
25
  ## Fallback vs abort
26
26
 
@@ -10,6 +10,9 @@ module Html2rss
10
10
  # Ordered list of concrete request strategies attempted by the :auto plan.
11
11
  CHAIN = %i[faraday botasaurus].freeze
12
12
 
13
+ # Deterministic HTTP statuses that indicate permanent failure rather than anti-bot challenges.
14
+ DETERMINISTIC_HTTP_STATUSES = Set[400, 404, 410, 422, 451].freeze
15
+
13
16
  # Error classes that should abort auto fallback immediately.
14
17
  NON_FALLBACK_ERRORS = [
15
18
  RequestService::UnknownStrategy,
@@ -18,7 +21,8 @@ module Html2rss
18
21
  RequestService::RequestBudgetExceeded,
19
22
  RequestService::PrivateNetworkDenied,
20
23
  RequestService::CrossOriginFollowUpDenied,
21
- RequestService::ResponseTooLarge
24
+ RequestService::ResponseTooLarge,
25
+ RequestService::RedirectLimitReached
22
26
  ].freeze
23
27
 
24
28
  ##
@@ -32,11 +36,20 @@ module Html2rss
32
36
  @last_response = nil
33
37
  @entry_resolution = nil
34
38
  @resolution_tried = false
39
+ @aborted = false
35
40
  end
36
41
 
37
42
  # @return [Boolean] true when a strategy already yielded items
38
43
  def succeeded? = !result.nil?
39
44
 
45
+ # @return [Boolean] whether attempts were aborted due to deterministic errors
46
+ def aborted? = @aborted
47
+
48
+ # @return [void]
49
+ def abort!
50
+ @aborted = true
51
+ end
52
+
40
53
  # @return [Boolean] whether entry resolution already ran for this chain
41
54
  def resolution_tried? = @resolution_tried
42
55
 
@@ -82,7 +95,7 @@ module Html2rss
82
95
  # @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
83
96
  # @param scrape_target [Html2rss::ScrapeTarget]
84
97
  # @return [void]
85
- # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
98
+ # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
86
99
  def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
87
100
  scrape_target:, admission_drops: {})
88
101
  @result = PipelineOutcome.new(
@@ -97,7 +110,6 @@ module Html2rss
97
110
  entry_resolution:
98
111
  )
99
112
  end
100
- # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
101
113
  end
102
114
 
103
115
  ##
@@ -133,7 +145,7 @@ module Html2rss
133
145
  AttemptState.new.tap do |state|
134
146
  strategies.each_with_index do |strategy, index|
135
147
  attempt(strategy:, next_strategy: strategies[index + 1], state:)
136
- break if state.succeeded?
148
+ break if state.succeeded? || state.aborted?
137
149
  end
138
150
  end
139
151
  end
@@ -161,7 +173,7 @@ module Html2rss
161
173
  nil
162
174
  end
163
175
 
164
- # rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
176
+ # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
165
177
  def process_response(response:, strategy:, next_strategy:, request_session:, state:)
166
178
  articles, dedup_dropped, admission_drops = articles_for(response:, request_session:)
167
179
  state.remember_response(response)
@@ -184,15 +196,24 @@ module Html2rss
184
196
  state:)
185
197
  end
186
198
 
199
+ if deterministic_status?(response)
200
+ state.abort!
201
+ Log.debug("#{self.class}: deterministic HTTP #{response.status}; aborting auto fallback")
202
+ return
203
+ end
204
+
187
205
  log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
188
206
  end
189
- # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
207
+
208
+ def deterministic_status?(response)
209
+ DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
210
+ end
190
211
 
191
212
  def articles_for(response:, request_session:)
192
213
  pipeline.deduplicated_articles(config:, response:, request_session:)
193
214
  end
194
215
 
195
- # rubocop:disable Metrics/ParameterLists -- success kwargs match PipelineOutcome
216
+ # rubocop:disable-next Metrics/ParameterLists -- success kwargs match PipelineOutcome
196
217
  def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
197
218
  attempt_count = state.attempts.size
198
219
  state.succeed!(
@@ -206,7 +227,6 @@ module Html2rss
206
227
  "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
207
228
  "budget_remaining=#{budget_remaining_label}")
208
229
  end
209
- # rubocop:enable Metrics/ParameterLists
210
230
 
211
231
  def finalize_failure(attempts:, response:)
212
232
  surface_category = surface_category_for(response)
@@ -219,7 +239,7 @@ module Html2rss
219
239
  PageRecon.surface_category_for(response:, url: response.url)
220
240
  end
221
241
 
222
- # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
242
+ # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
223
243
  def log_fallback_error(strategy:, next_strategy:, error:, request_session:)
224
244
  host = request_session.url.host
225
245
  detail = "host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
@@ -233,7 +253,6 @@ module Html2rss
233
253
  end
234
254
  Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message} #{detail}")
235
255
  end
236
- # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
237
256
 
238
257
  def log_info_fallback_zero_items(strategy:, next_strategy:, response:)
239
258
  Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items " \
@@ -31,28 +31,25 @@ module Html2rss
31
31
  pipeline_outcome_for(config)
32
32
  end
33
33
 
34
+ ##
35
+ # Runs the pipeline once and returns scrape outcome plus {FeedResult} without a second fetch.
36
+ #
37
+ # @return [Array(PipelineOutcome, FeedResult)]
38
+ def to_outcome_and_result
39
+ config = Config.from_hash(raw_config, params: raw_config[:params])
40
+ outcome = pipeline_outcome_for(config)
41
+ [outcome, feed_result_for(config, outcome)]
42
+ end
43
+
34
44
  ##
35
45
  # Runs the pipeline once and returns an opaque, Marshal-cacheable result.
36
46
  #
37
47
  # @return [Html2rss::FeedResult]
38
- # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- Status kwargs stay co-located with Channel
39
48
  def to_result
40
49
  config = Config.from_hash(raw_config, params: raw_config[:params])
41
50
  outcome = pipeline_outcome_for(config)
42
- channel = Channel.from_response(outcome.response, overrides: config.channel)
43
- status = Status.build(
44
- articles: outcome.articles,
45
- dedup_dropped: outcome.dedup_dropped,
46
- selected_strategy: outcome.selected_strategy,
47
- attempt_count: outcome.attempt_count,
48
- strategy_attempts: outcome.strategy_attempts,
49
- admission_drops: outcome.admission_drops,
50
- scrape_target: outcome.scrape_target,
51
- entry_resolution: outcome.entry_resolution
52
- )
53
- FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
51
+ feed_result_for(config, outcome)
54
52
  end
55
- # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
56
53
 
57
54
  # @api private Host seam for {AutoFallback} (and single-strategy path).
58
55
  # @param config [Html2rss::Config]
@@ -135,7 +132,7 @@ module Html2rss
135
132
  [selector + auto, admission_drops]
136
133
  end
137
134
 
138
- # rubocop:disable Metrics/MethodLength
135
+ # rubocop:disable-next Metrics/MethodLength
139
136
  def selector_articles(config:, response:, request_session:)
140
137
  return [] unless (selectors = config.selectors)
141
138
  return [] if response.feed_response?
@@ -154,7 +151,6 @@ module Html2rss
154
151
  end
155
152
  articles
156
153
  end
157
- # rubocop:enable Metrics/MethodLength
158
154
 
159
155
  # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
160
156
  def auto_source_articles(config:, response:, request_session:)
@@ -171,5 +167,20 @@ module Html2rss
171
167
  end
172
168
  [articles, {}]
173
169
  end
170
+
171
+ def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
172
+ channel = Channel.from_response(outcome.response, overrides: config.channel)
173
+ status = Status.build(
174
+ articles: outcome.articles,
175
+ dedup_dropped: outcome.dedup_dropped,
176
+ selected_strategy: outcome.selected_strategy,
177
+ attempt_count: outcome.attempt_count,
178
+ strategy_attempts: outcome.strategy_attempts,
179
+ admission_drops: outcome.admission_drops,
180
+ scrape_target: outcome.scrape_target,
181
+ entry_resolution: outcome.entry_resolution
182
+ )
183
+ FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
184
+ end
174
185
  end
175
186
  end
@@ -10,17 +10,17 @@ tournament on item count (feed-as-means) — intentional.
10
10
 
11
11
  ## Ownership
12
12
 
13
- | Concern | Owner |
14
- | --- | --- |
15
- | When to resolve (`articles:` required; floor / weak / NativeFeed ≥50%) | `FeedResolution::Policy` |
16
- | Candidate mix (1 feed + up to 4 listing; taxonomy nav; segment first-wins) | `FeedResolution::CandidateGenerator` |
17
- | Listing path lexicon (`LISTING_PATHS`) | `Syndication::CandidateCatalog` (consumed, not owned here) |
18
- | Cheap probe + score + winner pick | `FeedResolution::Probe` + `FeedResolution::Scorer` |
19
- | Typed entry_resolution options | `FeedResolution::Options` |
20
- | Wire-safe resolution diag (`applied` = tournament win, not sticky URL) | `FeedResolution::Diag` |
21
- | Public `call` → `Result`; retry orchestration (sticky `ScrapeTarget` only on `:succeeded`) | `FeedResolution` (`try_apply!`) |
22
- | Page surface for policy/scoring | `PageRecon::Assessment` + `Html2rss::SurfaceCategory` |
23
- | Native feed discover/parse | `Syndication` (not this module) |
13
+ | Concern | Owner |
14
+ | ------------------------------------------------------------------------------------------ | ---------------------------------------------------------- |
15
+ | When to resolve (`articles:` required; floor / weak / NativeFeed ≥50%) | `FeedResolution::Policy` |
16
+ | Candidate mix (1 feed + up to 4 listing; taxonomy nav; segment first-wins) | `FeedResolution::CandidateGenerator` |
17
+ | Listing path lexicon (`LISTING_PATHS`) | `Syndication::CandidateCatalog` (consumed, not owned here) |
18
+ | Cheap probe + score + winner pick | `FeedResolution::Probe` + `FeedResolution::Scorer` |
19
+ | Typed entry_resolution options | `FeedResolution::Options` |
20
+ | Wire-safe resolution diag (`applied` = tournament win, not sticky URL) | `FeedResolution::Diag` |
21
+ | Public `call` → `Result`; retry orchestration (sticky `ScrapeTarget` only on `:succeeded`) | `FeedResolution` (`try_apply!`) |
22
+ | Page surface for policy/scoring | `PageRecon::Assessment` + `Html2rss::SurfaceCategory` |
23
+ | Native feed discover/parse | `Syndication` (not this module) |
24
24
 
25
25
  ### CandidateGenerator mix
26
26