html2rss 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +82 -25
- data/html2rss.gemspec +0 -1
- data/lib/html2rss/article.rb +1 -2
- data/lib/html2rss/auto_source/README.md +8 -8
- data/lib/html2rss/auto_source/cleanup.rb +92 -1
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
- data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
- data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
- data/lib/html2rss/auto_source/scraper.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
- data/lib/html2rss/auto_source/segmenter.rb +1 -2
- data/lib/html2rss/auto_source.rb +25 -11
- data/lib/html2rss/batch.rb +169 -0
- data/lib/html2rss/capture/README.md +15 -6
- data/lib/html2rss/capture.rb +158 -15
- data/lib/html2rss/channel.rb +1 -2
- data/lib/html2rss/cli/probe_view.rb +66 -0
- data/lib/html2rss/cli/render.rb +124 -0
- data/lib/html2rss/cli/validate.rb +91 -0
- data/lib/html2rss/cli.rb +318 -193
- data/lib/html2rss/config/request_headers.rb +4 -14
- data/lib/html2rss/config/schema.rb +7 -5
- data/lib/html2rss/config/validator.rb +1 -0
- data/lib/html2rss/config.rb +59 -22
- data/lib/html2rss/doctor/botasaurus.rb +130 -0
- data/lib/html2rss/feed_builder/rss.rb +1 -2
- data/lib/html2rss/feed_pipeline/README.md +5 -5
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
- data/lib/html2rss/feed_pipeline.rb +27 -16
- data/lib/html2rss/feed_resolution/README.md +11 -11
- data/lib/html2rss/feed_resolution.rb +6 -12
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
- data/lib/html2rss/html/article_extractor.rb +4 -9
- data/lib/html2rss/html/feed_link.rb +4 -2
- data/lib/html2rss/html/navigator.rb +4 -39
- data/lib/html2rss/html/probe.rb +82 -0
- data/lib/html2rss/html/sst_article_extractor.rb +1 -2
- data/lib/html2rss/link_destination/noise_policy.rb +1 -2
- data/lib/html2rss/link_destination/path_classifier.rb +20 -19
- data/lib/html2rss/mcp/README.md +80 -0
- data/lib/html2rss/mcp/contract.rb +135 -13
- data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
- data/lib/html2rss/mcp/outcome.rb +112 -42
- data/lib/html2rss/mcp/runtime.rb +45 -0
- data/lib/html2rss/mcp/server/tools.rb +277 -0
- data/lib/html2rss/mcp/server.rb +91 -232
- data/lib/html2rss/mcp.rb +3 -0
- data/lib/html2rss/page_recon/README.md +69 -0
- data/lib/html2rss/page_recon/diagnostics.rb +211 -0
- data/lib/html2rss/page_recon.rb +56 -2
- data/lib/html2rss/recon.rb +252 -0
- data/lib/html2rss/request_service/blocked_surface.rb +1 -0
- data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
- data/lib/html2rss/request_service/policy.rb +1 -2
- data/lib/html2rss/request_service/response.rb +3 -4
- data/lib/html2rss/request_service/strategy.rb +1 -2
- data/lib/html2rss/request_service.rb +2 -0
- data/lib/html2rss/request_session.rb +1 -2
- data/lib/html2rss/scrape_target.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
- data/lib/html2rss/selectors/post_processors.rb +0 -1
- data/lib/html2rss/selectors.rb +33 -10
- data/lib/html2rss/sst/attrs.rb +1 -2
- data/lib/html2rss/sst/normalizer.rb +49 -11
- data/lib/html2rss/status.rb +3 -6
- data/lib/html2rss/syndication/README.md +6 -6
- data/lib/html2rss/syndication/discovery.rb +2 -4
- data/lib/html2rss/test/enhance_audit.rb +192 -0
- data/lib/html2rss/test.rb +458 -0
- data/lib/html2rss/url.rb +1 -2
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +157 -35
- data/schema/html2rss-config.schema.json +9 -29
- metadata +17 -18
- data/lib/html2rss/mcp/inspect.rb +0 -138
- data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
|
@@ -48,7 +48,7 @@ module Html2rss
|
|
|
48
48
|
##
|
|
49
49
|
# Orchestrates schema assembly from runtime validator contracts plus
|
|
50
50
|
# client-facing overlays.
|
|
51
|
-
# rubocop:disable Metrics/ClassLength -- overlay assembly stays in one builder
|
|
51
|
+
# rubocop:disable-next Metrics/ClassLength -- overlay assembly stays in one builder
|
|
52
52
|
class Builder
|
|
53
53
|
class << self
|
|
54
54
|
# @return [Hash{String => Object}] fully assembled JSON schema hash
|
|
@@ -110,9 +110,13 @@ module Html2rss
|
|
|
110
110
|
registry.keys.sort.to_h { |name| [name.to_s, registry.fetch(name).schema_doc] }
|
|
111
111
|
end
|
|
112
112
|
|
|
113
|
-
# rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
|
|
113
|
+
# rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
|
|
114
114
|
def overlay
|
|
115
115
|
items_schema = Html2rss::Config::SelectorsValidator::Items.new.schema.json_schema(loose: true)
|
|
116
|
+
items_schema[:properties][:enhance] = items_schema.fetch(:properties).fetch(:enhance).merge(
|
|
117
|
+
description: 'List-card enrichment: run Html::ArticleExtractor on each matched item node ' \
|
|
118
|
+
'to fill missing fields from the card HTML.'
|
|
119
|
+
)
|
|
116
120
|
items_schema[:properties][:pagination] = {
|
|
117
121
|
description: 'Pagination configuration or maximum page count integer.',
|
|
118
122
|
oneOf: [
|
|
@@ -166,7 +170,7 @@ module Html2rss
|
|
|
166
170
|
description: 'Selectors used to extract article attributes.',
|
|
167
171
|
properties: {
|
|
168
172
|
items: items_schema.merge(
|
|
169
|
-
description: 'Defines the items selector and
|
|
173
|
+
description: 'Defines the items selector and list-card enhance settings.'
|
|
170
174
|
),
|
|
171
175
|
enclosure: enclosure_schema.merge(
|
|
172
176
|
description: 'Describes enclosure extraction settings.'
|
|
@@ -183,7 +187,6 @@ module Html2rss
|
|
|
183
187
|
}
|
|
184
188
|
}
|
|
185
189
|
end
|
|
186
|
-
# rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Layout/LineLength
|
|
187
190
|
|
|
188
191
|
def reference_array(description)
|
|
189
192
|
{
|
|
@@ -197,7 +200,6 @@ module Html2rss
|
|
|
197
200
|
}
|
|
198
201
|
end
|
|
199
202
|
end
|
|
200
|
-
# rubocop:enable Metrics/ClassLength
|
|
201
203
|
|
|
202
204
|
##
|
|
203
205
|
# Wires extractor / post_process to thin oneOf $refs into the registry catalog.
|
|
@@ -19,6 +19,7 @@ module Html2rss
|
|
|
19
19
|
DIRECTORY_TOPICS = %w[
|
|
20
20
|
sports energy tech science news entertainment jobs finance
|
|
21
21
|
security travel environment consumer civic product research
|
|
22
|
+
health culture education rights transport
|
|
22
23
|
].freeze
|
|
23
24
|
|
|
24
25
|
# Contract for the top-level `channel` section.
|
data/lib/html2rss/config.rb
CHANGED
|
@@ -16,6 +16,32 @@ module Html2rss
|
|
|
16
16
|
# Sentinel to differentiate omitted params from explicit `nil`.
|
|
17
17
|
UNSET = Object.new.freeze
|
|
18
18
|
|
|
19
|
+
ValidationResult = Data.define(:success, :errors_hash)
|
|
20
|
+
|
|
21
|
+
##
|
|
22
|
+
# Duck-compatible with +Dry::Validation::Result+ for resolve-time outcomes
|
|
23
|
+
# (+#success?+, +#errors#to_h+). Used for parse failures so callers never see a Struct snowflake.
|
|
24
|
+
class ValidationResult
|
|
25
|
+
##
|
|
26
|
+
# @param message [String]
|
|
27
|
+
# @return [Html2rss::Config::ValidationResult]
|
|
28
|
+
def self.parse_failure(message)
|
|
29
|
+
new(success: false, errors_hash: { parse: [message] })
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
##
|
|
33
|
+
# @return [Boolean]
|
|
34
|
+
def success? = success
|
|
35
|
+
|
|
36
|
+
##
|
|
37
|
+
# @return [self]
|
|
38
|
+
def errors = self
|
|
39
|
+
|
|
40
|
+
##
|
|
41
|
+
# @return [Hash]
|
|
42
|
+
def to_h = errors_hash
|
|
43
|
+
end
|
|
44
|
+
|
|
19
45
|
class << self
|
|
20
46
|
##
|
|
21
47
|
# Returns the exported JSON Schema for html2rss configuration.
|
|
@@ -34,6 +60,22 @@ module Html2rss
|
|
|
34
60
|
pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
|
|
35
61
|
end
|
|
36
62
|
|
|
63
|
+
##
|
|
64
|
+
# Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it.
|
|
65
|
+
# The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.
|
|
66
|
+
#
|
|
67
|
+
# @param config_input [Hash, String] config hash, YAML string, or file path
|
|
68
|
+
# @param feed_name [String, nil] optional feed name for multi-feed files
|
|
69
|
+
# @param params [Hash] dynamic feed params
|
|
70
|
+
# @return [Array(Hash, Dry::Validation::Result, Html2rss::Config::ValidationResult)]
|
|
71
|
+
def resolve_and_validate(config_input, feed_name: nil, params: {})
|
|
72
|
+
param_arg = params.empty? ? UNSET : params
|
|
73
|
+
working = HashUtil.deep_dup(resolve_raw_hash(config_input, feed_name))
|
|
74
|
+
[working, validate(working, params: param_arg)]
|
|
75
|
+
rescue StandardError => error
|
|
76
|
+
[{}, ValidationResult.parse_failure(error.message)]
|
|
77
|
+
end
|
|
78
|
+
|
|
37
79
|
##
|
|
38
80
|
# Validates a configuration hash with the runtime validator.
|
|
39
81
|
#
|
|
@@ -59,22 +101,10 @@ module Html2rss
|
|
|
59
101
|
Schema.path
|
|
60
102
|
end
|
|
61
103
|
|
|
62
|
-
##
|
|
63
|
-
# Loads and validates a YAML configuration file.
|
|
64
|
-
#
|
|
65
|
-
# @param file [String] the YAML file to load
|
|
66
|
-
# @param feed_name [String, nil] optional feed name for multi-feed files
|
|
67
|
-
# @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
|
|
68
|
-
# @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
|
|
69
|
-
# @return [Dry::Validation::Result] validation result after defaults are applied
|
|
70
|
-
def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
|
|
71
|
-
validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
|
|
72
|
-
end
|
|
73
|
-
|
|
74
104
|
##
|
|
75
105
|
# Serializes a configuration hash to string-key YAML.
|
|
76
106
|
#
|
|
77
|
-
# This is the single serializer for CLI capture and MCP +
|
|
107
|
+
# This is the single serializer for CLI capture and MCP +capture+.
|
|
78
108
|
#
|
|
79
109
|
# @param hash [Hash] configuration hash (symbol or string keys)
|
|
80
110
|
# @return [String] YAML document without Ruby symbol-key prefixes
|
|
@@ -109,7 +139,7 @@ module Html2rss
|
|
|
109
139
|
# @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
|
|
110
140
|
# @return [Hash{Symbol => Object}] the configuration hash.
|
|
111
141
|
# @raise [ArgumentError] if the file doesn't exist or feed is not found.
|
|
112
|
-
# rubocop:disable Metrics/MethodLength
|
|
142
|
+
# rubocop:disable-next Metrics/MethodLength
|
|
113
143
|
def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
|
|
114
144
|
raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
|
|
115
145
|
raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
|
|
@@ -129,7 +159,6 @@ module Html2rss
|
|
|
129
159
|
|
|
130
160
|
MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
|
|
131
161
|
end
|
|
132
|
-
# rubocop:enable Metrics/MethodLength
|
|
133
162
|
|
|
134
163
|
##
|
|
135
164
|
# Processes the provided configuration hash, applying dynamic parameters if given,
|
|
@@ -216,6 +245,20 @@ module Html2rss
|
|
|
216
245
|
def prepare_for_validation(config)
|
|
217
246
|
Config::Preparer.new.call(HashUtil.deep_dup(config))
|
|
218
247
|
end
|
|
248
|
+
|
|
249
|
+
# @param config_input [Hash, String]
|
|
250
|
+
# @param feed_name [String, nil]
|
|
251
|
+
# @return [Hash]
|
|
252
|
+
def resolve_raw_hash(config_input, feed_name)
|
|
253
|
+
if config_input.is_a?(Hash)
|
|
254
|
+
config_input
|
|
255
|
+
elsif File.file?(config_input.to_s)
|
|
256
|
+
load_yaml(config_input.to_s, feed_name)
|
|
257
|
+
else
|
|
258
|
+
from_yaml(config_input.to_s)
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
private :resolve_raw_hash
|
|
219
262
|
end
|
|
220
263
|
|
|
221
264
|
##
|
|
@@ -270,11 +313,6 @@ module Html2rss
|
|
|
270
313
|
# @return [String]
|
|
271
314
|
def url = config.dig(:channel, :url)
|
|
272
315
|
|
|
273
|
-
##
|
|
274
|
-
# @deprecated Use {ScrapeTarget} for entry vs effective URLs after resolution.
|
|
275
|
-
# @return [String]
|
|
276
|
-
def scrape_url = url
|
|
277
|
-
|
|
278
316
|
# @return [String, nil] configured channel time zone
|
|
279
317
|
def time_zone = config.dig(:channel, :time_zone)
|
|
280
318
|
|
|
@@ -331,8 +369,7 @@ module Html2rss
|
|
|
331
369
|
def normalized_headers(validated_config)
|
|
332
370
|
validated_config[:headers] = RequestHeaders.normalize(
|
|
333
371
|
validated_config[:headers],
|
|
334
|
-
channel_language: validated_config.dig(:channel, :language)
|
|
335
|
-
url: validated_config.dig(:channel, :url)
|
|
372
|
+
channel_language: validated_config.dig(:channel, :language)
|
|
336
373
|
)
|
|
337
374
|
validated_config
|
|
338
375
|
end
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'faraday'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
module Html2rss
|
|
7
|
+
##
|
|
8
|
+
# CLI runtime health checks for optional scrape dependencies.
|
|
9
|
+
module Doctor
|
|
10
|
+
##
|
|
11
|
+
# Preflight checks for Botasaurus scrape-api connectivity.
|
|
12
|
+
module Botasaurus
|
|
13
|
+
##
|
|
14
|
+
# One named doctor check outcome.
|
|
15
|
+
Check = Data.define(:name, :ok, :detail) do
|
|
16
|
+
##
|
|
17
|
+
# @return [Hash{Symbol => Object}]
|
|
18
|
+
def to_h
|
|
19
|
+
{ name: name.to_s, ok:, detail: }
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
##
|
|
24
|
+
# Aggregate doctor command outcome.
|
|
25
|
+
Result = Data.define(:ok, :checks, :message) do
|
|
26
|
+
##
|
|
27
|
+
# @return [Hash{Symbol => Object}]
|
|
28
|
+
def to_h
|
|
29
|
+
{ ok:, message:, checks: checks.map(&:to_h) }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
module_function
|
|
34
|
+
|
|
35
|
+
ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
|
|
36
|
+
private_constant :ENV_VAR
|
|
37
|
+
|
|
38
|
+
##
|
|
39
|
+
# @param sample_url [String, nil] optional URL for a sample scrape smoke test
|
|
40
|
+
# @return [Result]
|
|
41
|
+
def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
|
|
42
|
+
env = env_check
|
|
43
|
+
checks = [env_check_record(env)]
|
|
44
|
+
return missing_env_failure(checks) unless checks.first.ok
|
|
45
|
+
|
|
46
|
+
checks << health_check_record(env[:base_url])
|
|
47
|
+
return failure(checks, 'Botasaurus health check failed.') unless checks.last.ok
|
|
48
|
+
|
|
49
|
+
checks << sample_scrape_record(sample_url) if sample_url
|
|
50
|
+
return failure(checks, 'Sample scrape failed.') if sample_url && !checks.last.ok
|
|
51
|
+
|
|
52
|
+
Log.info('doctor botasaurus: preflight passed')
|
|
53
|
+
Result.new(ok: true, checks:, message: 'Botasaurus preflight passed.')
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
##
|
|
57
|
+
# @return [Hash{Symbol => Object}]
|
|
58
|
+
def env_check
|
|
59
|
+
base = ENV.fetch(ENV_VAR, '').strip
|
|
60
|
+
{
|
|
61
|
+
configured: MCP::Runtime.botasaurus_configured?,
|
|
62
|
+
var: ENV_VAR,
|
|
63
|
+
base_url: base.empty? ? nil : base
|
|
64
|
+
}
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
##
|
|
68
|
+
# @param base_url [String]
|
|
69
|
+
# @return [Hash{Symbol => Object}]
|
|
70
|
+
def health_check(base_url)
|
|
71
|
+
uri = Url.for_channel(base_url)
|
|
72
|
+
client = Faraday.new(url: uri.to_s.chomp('/'), request: { timeout: 5 })
|
|
73
|
+
response = client.get('/health')
|
|
74
|
+
body = JSON.parse(response.body)
|
|
75
|
+
{ ok: response.status == 200, status: response.status, version: body['version'] }
|
|
76
|
+
rescue StandardError => error
|
|
77
|
+
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
##
|
|
81
|
+
# @param url [String]
|
|
82
|
+
# @return [Hash{Symbol => Object}]
|
|
83
|
+
def sample_scrape(url)
|
|
84
|
+
wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
|
|
85
|
+
{
|
|
86
|
+
ok: wire[:status].to_i.between?(200, 399),
|
|
87
|
+
status: wire[:status],
|
|
88
|
+
articles_count: wire[:articles_count]
|
|
89
|
+
}
|
|
90
|
+
rescue StandardError => error
|
|
91
|
+
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def env_check_record(env = env_check)
|
|
95
|
+
Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
|
|
96
|
+
end
|
|
97
|
+
module_function :env_check_record
|
|
98
|
+
private_class_method :env_check_record
|
|
99
|
+
|
|
100
|
+
def health_check_record(base_url)
|
|
101
|
+
return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
|
|
102
|
+
|
|
103
|
+
health = health_check(base_url)
|
|
104
|
+
Check.new(name: :health, ok: health[:ok], detail: health)
|
|
105
|
+
end
|
|
106
|
+
module_function :health_check_record
|
|
107
|
+
private_class_method :health_check_record
|
|
108
|
+
|
|
109
|
+
def sample_scrape_record(url)
|
|
110
|
+
sample = sample_scrape(url)
|
|
111
|
+
Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
|
|
112
|
+
end
|
|
113
|
+
module_function :sample_scrape_record
|
|
114
|
+
private_class_method :sample_scrape_record
|
|
115
|
+
|
|
116
|
+
def missing_env_failure(checks)
|
|
117
|
+
Log.info("doctor botasaurus: #{ENV_VAR} unset")
|
|
118
|
+
failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
|
|
119
|
+
end
|
|
120
|
+
module_function :missing_env_failure
|
|
121
|
+
private_class_method :missing_env_failure
|
|
122
|
+
|
|
123
|
+
def failure(checks, message)
|
|
124
|
+
Result.new(ok: false, checks:, message:)
|
|
125
|
+
end
|
|
126
|
+
module_function :failure
|
|
127
|
+
private_class_method :failure
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
@@ -96,7 +96,7 @@ module Html2rss
|
|
|
96
96
|
@stylesheets.map { |style| Stylesheet.new(**style) }
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
-
# rubocop:disable Metrics/AbcSize
|
|
99
|
+
# rubocop:disable-next Metrics/AbcSize
|
|
100
100
|
def make_channel(maker)
|
|
101
101
|
channel_maker = maker.channel
|
|
102
102
|
%i[language title description ttl].each do |key|
|
|
@@ -111,7 +111,6 @@ module Html2rss
|
|
|
111
111
|
|
|
112
112
|
make_image(maker.image) if channel.image
|
|
113
113
|
end
|
|
114
|
-
# rubocop:enable Metrics/AbcSize
|
|
115
114
|
|
|
116
115
|
def make_image(image_maker)
|
|
117
116
|
image_maker.url = channel.image.to_s
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# FeedPipeline — `auto` request strategy
|
|
2
2
|
|
|
3
|
-
`:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `
|
|
3
|
+
`:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape` / `capture`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
|
|
4
4
|
|
|
5
5
|
Use `:auto` when you want Faraday first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`faraday`, `botasaurus`, `local_file`) when you need a single transport.
|
|
6
6
|
|
|
@@ -17,10 +17,10 @@ There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want
|
|
|
17
17
|
|
|
18
18
|
## Surfaces
|
|
19
19
|
|
|
20
|
-
| Surface
|
|
21
|
-
|
|
22
|
-
| Gem / CLI
|
|
23
|
-
| MCP `
|
|
20
|
+
| Surface | `:auto` behavior |
|
|
21
|
+
| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
|
|
22
|
+
| Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`faraday` → `botasaurus`) |
|
|
23
|
+
| MCP `inspect` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to Faraday (pin `botasaurus` when you need browser rendering) |
|
|
24
24
|
|
|
25
25
|
## Fallback vs abort
|
|
26
26
|
|
|
@@ -10,6 +10,9 @@ module Html2rss
|
|
|
10
10
|
# Ordered list of concrete request strategies attempted by the :auto plan.
|
|
11
11
|
CHAIN = %i[faraday botasaurus].freeze
|
|
12
12
|
|
|
13
|
+
# Deterministic HTTP statuses that indicate permanent failure rather than anti-bot challenges.
|
|
14
|
+
DETERMINISTIC_HTTP_STATUSES = Set[400, 404, 410, 422, 451].freeze
|
|
15
|
+
|
|
13
16
|
# Error classes that should abort auto fallback immediately.
|
|
14
17
|
NON_FALLBACK_ERRORS = [
|
|
15
18
|
RequestService::UnknownStrategy,
|
|
@@ -18,7 +21,8 @@ module Html2rss
|
|
|
18
21
|
RequestService::RequestBudgetExceeded,
|
|
19
22
|
RequestService::PrivateNetworkDenied,
|
|
20
23
|
RequestService::CrossOriginFollowUpDenied,
|
|
21
|
-
RequestService::ResponseTooLarge
|
|
24
|
+
RequestService::ResponseTooLarge,
|
|
25
|
+
RequestService::RedirectLimitReached
|
|
22
26
|
].freeze
|
|
23
27
|
|
|
24
28
|
##
|
|
@@ -32,11 +36,20 @@ module Html2rss
|
|
|
32
36
|
@last_response = nil
|
|
33
37
|
@entry_resolution = nil
|
|
34
38
|
@resolution_tried = false
|
|
39
|
+
@aborted = false
|
|
35
40
|
end
|
|
36
41
|
|
|
37
42
|
# @return [Boolean] true when a strategy already yielded items
|
|
38
43
|
def succeeded? = !result.nil?
|
|
39
44
|
|
|
45
|
+
# @return [Boolean] whether attempts were aborted due to deterministic errors
|
|
46
|
+
def aborted? = @aborted
|
|
47
|
+
|
|
48
|
+
# @return [void]
|
|
49
|
+
def abort!
|
|
50
|
+
@aborted = true
|
|
51
|
+
end
|
|
52
|
+
|
|
40
53
|
# @return [Boolean] whether entry resolution already ran for this chain
|
|
41
54
|
def resolution_tried? = @resolution_tried
|
|
42
55
|
|
|
@@ -82,7 +95,7 @@ module Html2rss
|
|
|
82
95
|
# @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
|
|
83
96
|
# @param scrape_target [Html2rss::ScrapeTarget]
|
|
84
97
|
# @return [void]
|
|
85
|
-
# rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
|
|
98
|
+
# rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
|
|
86
99
|
def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
|
|
87
100
|
scrape_target:, admission_drops: {})
|
|
88
101
|
@result = PipelineOutcome.new(
|
|
@@ -97,7 +110,6 @@ module Html2rss
|
|
|
97
110
|
entry_resolution:
|
|
98
111
|
)
|
|
99
112
|
end
|
|
100
|
-
# rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
|
|
101
113
|
end
|
|
102
114
|
|
|
103
115
|
##
|
|
@@ -133,7 +145,7 @@ module Html2rss
|
|
|
133
145
|
AttemptState.new.tap do |state|
|
|
134
146
|
strategies.each_with_index do |strategy, index|
|
|
135
147
|
attempt(strategy:, next_strategy: strategies[index + 1], state:)
|
|
136
|
-
break if state.succeeded?
|
|
148
|
+
break if state.succeeded? || state.aborted?
|
|
137
149
|
end
|
|
138
150
|
end
|
|
139
151
|
end
|
|
@@ -161,7 +173,7 @@ module Html2rss
|
|
|
161
173
|
nil
|
|
162
174
|
end
|
|
163
175
|
|
|
164
|
-
# rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
|
|
176
|
+
# rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize -- extract + resolution delegate
|
|
165
177
|
def process_response(response:, strategy:, next_strategy:, request_session:, state:)
|
|
166
178
|
articles, dedup_dropped, admission_drops = articles_for(response:, request_session:)
|
|
167
179
|
state.remember_response(response)
|
|
@@ -184,15 +196,24 @@ module Html2rss
|
|
|
184
196
|
state:)
|
|
185
197
|
end
|
|
186
198
|
|
|
199
|
+
if deterministic_status?(response)
|
|
200
|
+
state.abort!
|
|
201
|
+
Log.debug("#{self.class}: deterministic HTTP #{response.status}; aborting auto fallback")
|
|
202
|
+
return
|
|
203
|
+
end
|
|
204
|
+
|
|
187
205
|
log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
|
|
188
206
|
end
|
|
189
|
-
|
|
207
|
+
|
|
208
|
+
def deterministic_status?(response)
|
|
209
|
+
DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
|
|
210
|
+
end
|
|
190
211
|
|
|
191
212
|
def articles_for(response:, request_session:)
|
|
192
213
|
pipeline.deduplicated_articles(config:, response:, request_session:)
|
|
193
214
|
end
|
|
194
215
|
|
|
195
|
-
# rubocop:disable Metrics/ParameterLists -- success kwargs match PipelineOutcome
|
|
216
|
+
# rubocop:disable-next Metrics/ParameterLists -- success kwargs match PipelineOutcome
|
|
196
217
|
def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
|
|
197
218
|
attempt_count = state.attempts.size
|
|
198
219
|
state.succeed!(
|
|
@@ -206,7 +227,6 @@ module Html2rss
|
|
|
206
227
|
"host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
|
|
207
228
|
"budget_remaining=#{budget_remaining_label}")
|
|
208
229
|
end
|
|
209
|
-
# rubocop:enable Metrics/ParameterLists
|
|
210
230
|
|
|
211
231
|
def finalize_failure(attempts:, response:)
|
|
212
232
|
surface_category = surface_category_for(response)
|
|
@@ -219,7 +239,7 @@ module Html2rss
|
|
|
219
239
|
PageRecon.surface_category_for(response:, url: response.url)
|
|
220
240
|
end
|
|
221
241
|
|
|
222
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
242
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
|
|
223
243
|
def log_fallback_error(strategy:, next_strategy:, error:, request_session:)
|
|
224
244
|
host = request_session.url.host
|
|
225
245
|
detail = "host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
|
|
@@ -233,7 +253,6 @@ module Html2rss
|
|
|
233
253
|
end
|
|
234
254
|
Log.debug("#{self.class}: strategy=#{strategy} error=#{error.class}: #{error.message} #{detail}")
|
|
235
255
|
end
|
|
236
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
237
256
|
|
|
238
257
|
def log_info_fallback_zero_items(strategy:, next_strategy:, response:)
|
|
239
258
|
Log.info("#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items " \
|
|
@@ -31,28 +31,25 @@ module Html2rss
|
|
|
31
31
|
pipeline_outcome_for(config)
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
##
|
|
35
|
+
# Runs the pipeline once and returns scrape outcome plus {FeedResult} without a second fetch.
|
|
36
|
+
#
|
|
37
|
+
# @return [Array(PipelineOutcome, FeedResult)]
|
|
38
|
+
def to_outcome_and_result
|
|
39
|
+
config = Config.from_hash(raw_config, params: raw_config[:params])
|
|
40
|
+
outcome = pipeline_outcome_for(config)
|
|
41
|
+
[outcome, feed_result_for(config, outcome)]
|
|
42
|
+
end
|
|
43
|
+
|
|
34
44
|
##
|
|
35
45
|
# Runs the pipeline once and returns an opaque, Marshal-cacheable result.
|
|
36
46
|
#
|
|
37
47
|
# @return [Html2rss::FeedResult]
|
|
38
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- Status kwargs stay co-located with Channel
|
|
39
48
|
def to_result
|
|
40
49
|
config = Config.from_hash(raw_config, params: raw_config[:params])
|
|
41
50
|
outcome = pipeline_outcome_for(config)
|
|
42
|
-
|
|
43
|
-
status = Status.build(
|
|
44
|
-
articles: outcome.articles,
|
|
45
|
-
dedup_dropped: outcome.dedup_dropped,
|
|
46
|
-
selected_strategy: outcome.selected_strategy,
|
|
47
|
-
attempt_count: outcome.attempt_count,
|
|
48
|
-
strategy_attempts: outcome.strategy_attempts,
|
|
49
|
-
admission_drops: outcome.admission_drops,
|
|
50
|
-
scrape_target: outcome.scrape_target,
|
|
51
|
-
entry_resolution: outcome.entry_resolution
|
|
52
|
-
)
|
|
53
|
-
FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
|
|
51
|
+
feed_result_for(config, outcome)
|
|
54
52
|
end
|
|
55
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
56
53
|
|
|
57
54
|
# @api private Host seam for {AutoFallback} (and single-strategy path).
|
|
58
55
|
# @param config [Html2rss::Config]
|
|
@@ -135,7 +132,7 @@ module Html2rss
|
|
|
135
132
|
[selector + auto, admission_drops]
|
|
136
133
|
end
|
|
137
134
|
|
|
138
|
-
# rubocop:disable Metrics/MethodLength
|
|
135
|
+
# rubocop:disable-next Metrics/MethodLength
|
|
139
136
|
def selector_articles(config:, response:, request_session:)
|
|
140
137
|
return [] unless (selectors = config.selectors)
|
|
141
138
|
return [] if response.feed_response?
|
|
@@ -154,7 +151,6 @@ module Html2rss
|
|
|
154
151
|
end
|
|
155
152
|
articles
|
|
156
153
|
end
|
|
157
|
-
# rubocop:enable Metrics/MethodLength
|
|
158
154
|
|
|
159
155
|
# @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
|
|
160
156
|
def auto_source_articles(config:, response:, request_session:)
|
|
@@ -171,5 +167,20 @@ module Html2rss
|
|
|
171
167
|
end
|
|
172
168
|
[articles, {}]
|
|
173
169
|
end
|
|
170
|
+
|
|
171
|
+
def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
|
|
172
|
+
channel = Channel.from_response(outcome.response, overrides: config.channel)
|
|
173
|
+
status = Status.build(
|
|
174
|
+
articles: outcome.articles,
|
|
175
|
+
dedup_dropped: outcome.dedup_dropped,
|
|
176
|
+
selected_strategy: outcome.selected_strategy,
|
|
177
|
+
attempt_count: outcome.attempt_count,
|
|
178
|
+
strategy_attempts: outcome.strategy_attempts,
|
|
179
|
+
admission_drops: outcome.admission_drops,
|
|
180
|
+
scrape_target: outcome.scrape_target,
|
|
181
|
+
entry_resolution: outcome.entry_resolution
|
|
182
|
+
)
|
|
183
|
+
FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
|
|
184
|
+
end
|
|
174
185
|
end
|
|
175
186
|
end
|
|
@@ -10,17 +10,17 @@ tournament on item count (feed-as-means) — intentional.
|
|
|
10
10
|
|
|
11
11
|
## Ownership
|
|
12
12
|
|
|
13
|
-
| Concern
|
|
14
|
-
|
|
|
15
|
-
| When to resolve (`articles:` required; floor / weak / NativeFeed ≥50%)
|
|
16
|
-
| Candidate mix (1 feed + up to 4 listing; taxonomy nav; segment first-wins)
|
|
17
|
-
| Listing path lexicon (`LISTING_PATHS`)
|
|
18
|
-
| Cheap probe + score + winner pick
|
|
19
|
-
| Typed entry_resolution options
|
|
20
|
-
| Wire-safe resolution diag (`applied` = tournament win, not sticky URL)
|
|
21
|
-
| Public `call` → `Result`; retry orchestration (sticky `ScrapeTarget` only on `:succeeded`) | `FeedResolution` (`try_apply!`)
|
|
22
|
-
| Page surface for policy/scoring
|
|
23
|
-
| Native feed discover/parse
|
|
13
|
+
| Concern | Owner |
|
|
14
|
+
| ------------------------------------------------------------------------------------------ | ---------------------------------------------------------- |
|
|
15
|
+
| When to resolve (`articles:` required; floor / weak / NativeFeed ≥50%) | `FeedResolution::Policy` |
|
|
16
|
+
| Candidate mix (1 feed + up to 4 listing; taxonomy nav; segment first-wins) | `FeedResolution::CandidateGenerator` |
|
|
17
|
+
| Listing path lexicon (`LISTING_PATHS`) | `Syndication::CandidateCatalog` (consumed, not owned here) |
|
|
18
|
+
| Cheap probe + score + winner pick | `FeedResolution::Probe` + `FeedResolution::Scorer` |
|
|
19
|
+
| Typed entry_resolution options | `FeedResolution::Options` |
|
|
20
|
+
| Wire-safe resolution diag (`applied` = tournament win, not sticky URL) | `FeedResolution::Diag` |
|
|
21
|
+
| Public `call` → `Result`; retry orchestration (sticky `ScrapeTarget` only on `:succeeded`) | `FeedResolution` (`try_apply!`) |
|
|
22
|
+
| Page surface for policy/scoring | `PageRecon::Assessment` + `Html2rss::SurfaceCategory` |
|
|
23
|
+
| Native feed discover/parse | `Syndication` (not this module) |
|
|
24
24
|
|
|
25
25
|
### CandidateGenerator mix
|
|
26
26
|
|