html2rss 0.29.1 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -8
- data/html2rss.gemspec +1 -3
- data/lib/html2rss/article/deduplicator.rb +2 -4
- data/lib/html2rss/article.rb +16 -39
- data/lib/html2rss/auto_source/README.md +1 -1
- data/lib/html2rss/auto_source/cleanup.rb +15 -6
- data/lib/html2rss/auto_source/scraper/html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
- data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
- data/lib/html2rss/auto_source/scraper.rb +4 -14
- data/lib/html2rss/auto_source/segment.rb +7 -4
- data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
- data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
- data/lib/html2rss/auto_source/segmenter.rb +5 -5
- data/lib/html2rss/auto_source.rb +2 -2
- data/lib/html2rss/batch.rb +6 -4
- data/lib/html2rss/capture/README.md +7 -5
- data/lib/html2rss/capture.rb +68 -36
- data/lib/html2rss/channel.rb +3 -3
- data/lib/html2rss/cli/probe_view.rb +5 -3
- data/lib/html2rss/cli/render.rb +6 -5
- data/lib/html2rss/cli/validate.rb +28 -27
- data/lib/html2rss/cli.rb +1 -1
- data/lib/html2rss/config/issue_mapper.rb +108 -0
- data/lib/html2rss/config/request_headers.rb +6 -4
- data/lib/html2rss/config/schema.rb +16 -1
- data/lib/html2rss/config/selectors_validator.rb +92 -70
- data/lib/html2rss/config/validation_issue.rb +60 -0
- data/lib/html2rss/config/validation_report.rb +55 -0
- data/lib/html2rss/config/validator.rb +10 -6
- data/lib/html2rss/config.rb +35 -101
- data/lib/html2rss/defaults.rb +42 -30
- data/lib/html2rss/doctor/botasaurus.rb +19 -9
- data/lib/html2rss/doctor.rb +8 -0
- data/lib/html2rss/error.rb +1 -1
- data/lib/html2rss/feed_builder/rss.rb +9 -14
- data/lib/html2rss/feed_pipeline/README.md +8 -8
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +1 -1
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +7 -3
- data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
- data/lib/html2rss/feed_resolution/diag.rb +3 -1
- data/lib/html2rss/feed_resolution/options.rb +3 -1
- data/lib/html2rss/feed_resolution/policy.rb +2 -2
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +2 -4
- data/lib/html2rss/html/feed_link.rb +3 -1
- data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
- data/lib/html2rss/html/navigator.rb +10 -10
- data/lib/html2rss/html/rendering/description_builder.rb +1 -1
- data/lib/html2rss/html/sst_article_extractor.rb +8 -5
- data/lib/html2rss/link_destination/destination_facts.rb +4 -2
- data/lib/html2rss/link_destination/noise_policy.rb +10 -1
- data/lib/html2rss/link_destination/path_classifier.rb +45 -38
- data/lib/html2rss/mcp/README.md +5 -3
- data/lib/html2rss/mcp/config_argument.rb +3 -1
- data/lib/html2rss/mcp/contract.rb +15 -25
- data/lib/html2rss/mcp/outcome/playbook.rb +7 -7
- data/lib/html2rss/mcp/outcome.rb +36 -43
- data/lib/html2rss/mcp/server/tools.rb +112 -132
- data/lib/html2rss/mcp/server.rb +27 -19
- data/lib/html2rss/page_recon/README.md +5 -5
- data/lib/html2rss/page_recon/diagnostics.rb +71 -81
- data/lib/html2rss/page_recon.rb +9 -5
- data/lib/html2rss/recon.rb +20 -31
- data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
- data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
- data/lib/html2rss/request_service/compressed_body.rb +17 -9
- data/lib/html2rss/request_service/httpx_strategy.rb +239 -0
- data/lib/html2rss/request_service/network_guard.rb +7 -5
- data/lib/html2rss/request_service/response.rb +1 -2
- data/lib/html2rss/request_service/response_guard.rb +0 -16
- data/lib/html2rss/request_service/strategy.rb +22 -11
- data/lib/html2rss/request_service.rb +46 -85
- data/lib/html2rss/scoring/container_assessor.rb +3 -1
- data/lib/html2rss/scoring/engine.rb +8 -8
- data/lib/html2rss/scoring/observation.rb +5 -3
- data/lib/html2rss/scoring/ranked_segment.rb +3 -1
- data/lib/html2rss/scrape_target.rb +3 -1
- data/lib/html2rss/selector_candidates.rb +234 -0
- data/lib/html2rss/selectors/README.md +64 -0
- data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
- data/lib/html2rss/selectors/extractors/href.rb +15 -16
- data/lib/html2rss/selectors/extractors/html.rb +8 -9
- data/lib/html2rss/selectors/extractors/static.rb +12 -14
- data/lib/html2rss/selectors/extractors/text.rb +8 -9
- data/lib/html2rss/selectors/extractors.rb +33 -20
- data/lib/html2rss/selectors/item_env.rb +31 -0
- data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
- data/lib/html2rss/selectors/option_spec.rb +140 -0
- data/lib/html2rss/selectors/post_processors/base.rb +79 -31
- data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
- data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
- data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
- data/lib/html2rss/selectors/post_processors/template.rb +23 -19
- data/lib/html2rss/selectors/post_processors.rb +6 -5
- data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
- data/lib/html2rss/selectors/step_env.rb +24 -0
- data/lib/html2rss/selectors.rb +231 -142
- data/lib/html2rss/sst/attrs.rb +5 -3
- data/lib/html2rss/sst/document.rb +3 -1
- data/lib/html2rss/sst/index.rb +3 -1
- data/lib/html2rss/sst/node.rb +29 -10
- data/lib/html2rss/sst/normalizer.rb +12 -16
- data/lib/html2rss/sst/tags.rb +7 -7
- data/lib/html2rss/status.rb +13 -8
- data/lib/html2rss/surface_category.rb +14 -32
- data/lib/html2rss/syndication/parser.rb +2 -2
- data/lib/html2rss/test/enhance_audit.rb +1 -3
- data/lib/html2rss/test/policy.rb +106 -0
- data/lib/html2rss/test.rb +126 -172
- data/lib/html2rss/url.rb +25 -4
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +27 -19
- data/lib/tasks/config_schema.rake +2 -2
- data/schema/html2rss-config.schema.json +11 -8
- metadata +19 -44
- data/lib/html2rss/request_service/faraday_strategy.rb +0 -233
- data/lib/html2rss/selectors/item_scope.rb +0 -30
data/lib/html2rss/config.rb
CHANGED
|
@@ -13,53 +13,8 @@ module Html2rss
|
|
|
13
13
|
class Config # rubocop:disable Metrics/ClassLength
|
|
14
14
|
# Raised when a configuration hash fails runtime validation.
|
|
15
15
|
class InvalidConfig < Html2rss::Error; end
|
|
16
|
-
# Sentinel to differentiate omitted params from explicit `nil`.
|
|
17
|
-
UNSET = Object.new.freeze
|
|
18
|
-
|
|
19
|
-
ValidationResult = Data.define(:success, :errors_hash)
|
|
20
|
-
|
|
21
|
-
##
|
|
22
|
-
# Duck-compatible with +Dry::Validation::Result+ for resolve-time outcomes
|
|
23
|
-
# (+#success?+, +#errors#to_h+). Used for parse failures so callers never see a Struct snowflake.
|
|
24
|
-
class ValidationResult
|
|
25
|
-
##
|
|
26
|
-
# @param message [String]
|
|
27
|
-
# @return [Html2rss::Config::ValidationResult]
|
|
28
|
-
def self.parse_failure(message)
|
|
29
|
-
new(success: false, errors_hash: { parse: [message] })
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
##
|
|
33
|
-
# @return [Boolean]
|
|
34
|
-
def success? = success
|
|
35
|
-
|
|
36
|
-
##
|
|
37
|
-
# @return [self]
|
|
38
|
-
def errors = self
|
|
39
|
-
|
|
40
|
-
##
|
|
41
|
-
# @return [Hash]
|
|
42
|
-
def to_h = errors_hash
|
|
43
|
-
end
|
|
44
16
|
|
|
45
17
|
class << self
|
|
46
|
-
##
|
|
47
|
-
# Returns the exported JSON Schema for html2rss configuration.
|
|
48
|
-
#
|
|
49
|
-
# @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
|
|
50
|
-
def json_schema
|
|
51
|
-
Schema.json_schema
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
##
|
|
55
|
-
# Returns the exported JSON Schema as JSON.
|
|
56
|
-
#
|
|
57
|
-
# @param pretty [Boolean] whether to pretty-print the JSON output
|
|
58
|
-
# @return [String] serialized JSON Schema
|
|
59
|
-
def json_schema_json(pretty: true)
|
|
60
|
-
pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
|
|
61
|
-
end
|
|
62
|
-
|
|
63
18
|
##
|
|
64
19
|
# Resolves a Hash, YAML file path, or YAML string to a working config Hash and validates it.
|
|
65
20
|
# The returned Hash is a deep copy — callers may stamp strategy/params without mutating input.
|
|
@@ -67,13 +22,14 @@ module Html2rss
|
|
|
67
22
|
# @param config_input [Hash, String] config hash, YAML string, or file path
|
|
68
23
|
# @param feed_name [String, nil] optional feed name for multi-feed files
|
|
69
24
|
# @param params [Hash] dynamic feed params
|
|
70
|
-
# @return [Array(Hash,
|
|
25
|
+
# @return [Array(Hash, Html2rss::Config::ValidationReport)]
|
|
71
26
|
def resolve_and_validate(config_input, feed_name: nil, params: {})
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
[
|
|
75
|
-
|
|
76
|
-
|
|
27
|
+
raw = resolve_raw_hash(config_input, feed_name)
|
|
28
|
+
rescue Psych::Exception, ArgumentError => error
|
|
29
|
+
[{}, IssueMapper.parse_failure(error.message)]
|
|
30
|
+
else
|
|
31
|
+
working = HashUtil.deep_dup(raw)
|
|
32
|
+
[working, validate(working, params:)]
|
|
77
33
|
end
|
|
78
34
|
|
|
79
35
|
##
|
|
@@ -81,24 +37,15 @@ module Html2rss
|
|
|
81
37
|
#
|
|
82
38
|
# @param config [Hash{Symbol => Object}] the configuration hash
|
|
83
39
|
# @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting
|
|
84
|
-
# @return [
|
|
85
|
-
def validate(config, params:
|
|
40
|
+
# @return [Html2rss::Config::ValidationReport]
|
|
41
|
+
def validate(config, params: {})
|
|
86
42
|
prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
|
|
87
|
-
|
|
88
|
-
Validator.new.call(prepared_config)
|
|
43
|
+
IssueMapper.from(Validator.new.call(prepared_config), values: prepared_config)
|
|
89
44
|
rescue DynamicParams::ParamsMissing => error
|
|
90
45
|
prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
|
|
91
46
|
prepared_config[:dynamic_params_error] = error.message
|
|
92
47
|
|
|
93
|
-
Validator.new.call(prepared_config)
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
##
|
|
97
|
-
# Returns the packaged JSON Schema file path.
|
|
98
|
-
#
|
|
99
|
-
# @return [String] absolute path to the packaged JSON Schema file
|
|
100
|
-
def schema_path
|
|
101
|
-
Schema.path
|
|
48
|
+
IssueMapper.from(Validator.new.call(prepared_config), values: prepared_config)
|
|
102
49
|
end
|
|
103
50
|
|
|
104
51
|
##
|
|
@@ -167,7 +114,7 @@ module Html2rss
|
|
|
167
114
|
# @param config [Hash{Symbol => Object}] the configuration hash.
|
|
168
115
|
# @param params [Hash{Symbol => Object, Hash{String => Object, nil}}] dynamic parameters for string formatting.
|
|
169
116
|
# @return [Html2rss::Config] the configuration object.
|
|
170
|
-
def from_hash(config, params:
|
|
117
|
+
def from_hash(config, params: {})
|
|
171
118
|
new(resolve_effective_config(config, params:))
|
|
172
119
|
end
|
|
173
120
|
|
|
@@ -214,6 +161,20 @@ module Html2rss
|
|
|
214
161
|
Html2rss.defaults.default_strategy || :auto
|
|
215
162
|
end
|
|
216
163
|
|
|
164
|
+
##
|
|
165
|
+
# Normalizes raw config input with defaults.
|
|
166
|
+
#
|
|
167
|
+
# @param config [Hash{Symbol => Object}] raw config input
|
|
168
|
+
# @return [Hash{Symbol => Object}] config with defaults applied
|
|
169
|
+
def prepare_defaults(config)
|
|
170
|
+
config = HashUtil.deep_merge(default_config, config)
|
|
171
|
+
config = HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config) if config[:selectors]
|
|
172
|
+
if config[:auto_source]
|
|
173
|
+
config = HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
|
|
174
|
+
end
|
|
175
|
+
config
|
|
176
|
+
end
|
|
177
|
+
|
|
217
178
|
private
|
|
218
179
|
|
|
219
180
|
def default_request_config
|
|
@@ -227,7 +188,7 @@ module Html2rss
|
|
|
227
188
|
def resolve_effective_config(config, params:)
|
|
228
189
|
cfg = HashUtil.deep_symbolize_keys(config, context: 'config')
|
|
229
190
|
p = parameter_defaults(cfg)
|
|
230
|
-
p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.
|
|
191
|
+
p.merge!(HashUtil.deep_symbolize_keys(params, context: 'params')) unless params.nil?
|
|
231
192
|
|
|
232
193
|
cfg[:headers] = DynamicParams.call(cfg[:headers], p) if cfg[:headers]
|
|
233
194
|
cfg[:channel] = DynamicParams.call(cfg[:channel], p) if cfg[:channel]
|
|
@@ -243,7 +204,7 @@ module Html2rss
|
|
|
243
204
|
end
|
|
244
205
|
|
|
245
206
|
def prepare_for_validation(config)
|
|
246
|
-
|
|
207
|
+
prepare_defaults(HashUtil.deep_dup(config))
|
|
247
208
|
end
|
|
248
209
|
|
|
249
210
|
# @param config_input [Hash, String]
|
|
@@ -270,7 +231,7 @@ module Html2rss
|
|
|
270
231
|
# @raise [InvalidConfig] if the configuration fails validation.
|
|
271
232
|
def initialize(config)
|
|
272
233
|
@request_controls = RequestControls.from_config(config)
|
|
273
|
-
prepared_config =
|
|
234
|
+
prepared_config = self.class.prepare_defaults(config.dup)
|
|
274
235
|
validated_config = validated_config_for(prepared_config)
|
|
275
236
|
|
|
276
237
|
@config = validated_config.freeze
|
|
@@ -328,42 +289,15 @@ module Html2rss
|
|
|
328
289
|
|
|
329
290
|
attr_reader :config
|
|
330
291
|
|
|
331
|
-
# Normalizes raw config input before validation.
|
|
332
|
-
class Preparer
|
|
333
|
-
##
|
|
334
|
-
# @param config [Hash{Symbol => Object}] raw config input
|
|
335
|
-
# @return [Hash{Symbol => Object}] config with defaults applied
|
|
336
|
-
def call(config)
|
|
337
|
-
config = config.dup if config.frozen?
|
|
338
|
-
|
|
339
|
-
config = apply_default_config(config)
|
|
340
|
-
config = apply_default_selectors_config(config) if config[:selectors]
|
|
341
|
-
config = apply_default_auto_source_config(config) if config[:auto_source]
|
|
342
|
-
|
|
343
|
-
config
|
|
344
|
-
end
|
|
345
|
-
|
|
346
|
-
private
|
|
347
|
-
|
|
348
|
-
def apply_default_config(config)
|
|
349
|
-
HashUtil.deep_merge(Config.default_config, config)
|
|
350
|
-
end
|
|
351
|
-
|
|
352
|
-
def apply_default_selectors_config(config)
|
|
353
|
-
HashUtil.deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
|
|
354
|
-
end
|
|
355
|
-
|
|
356
|
-
def apply_default_auto_source_config(config)
|
|
357
|
-
HashUtil.deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
|
|
358
|
-
end
|
|
359
|
-
end
|
|
360
|
-
|
|
361
292
|
def validated_config_for(config)
|
|
362
|
-
|
|
293
|
+
dry = Validator.new.call(config)
|
|
363
294
|
|
|
364
|
-
|
|
295
|
+
unless dry.success?
|
|
296
|
+
report = IssueMapper.from(dry, values: config)
|
|
297
|
+
raise InvalidConfig, "Invalid configuration: #{report}"
|
|
298
|
+
end
|
|
365
299
|
|
|
366
|
-
normalized_headers(
|
|
300
|
+
normalized_headers(dry.to_h)
|
|
367
301
|
end
|
|
368
302
|
|
|
369
303
|
def normalized_headers(validated_config)
|
data/lib/html2rss/defaults.rb
CHANGED
|
@@ -4,13 +4,20 @@ module Html2rss
|
|
|
4
4
|
##
|
|
5
5
|
# Global defaults for the Html2rss gem.
|
|
6
6
|
class Defaults
|
|
7
|
-
#
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
# Named Logger severities accepted by {#log_level=}.
|
|
8
|
+
LOG_LEVELS = {
|
|
9
|
+
debug: Logger::DEBUG,
|
|
10
|
+
info: Logger::INFO,
|
|
11
|
+
warn: Logger::WARN,
|
|
12
|
+
error: Logger::ERROR,
|
|
13
|
+
fatal: Logger::FATAL,
|
|
14
|
+
unknown: Logger::UNKNOWN
|
|
15
|
+
}.freeze
|
|
16
|
+
|
|
17
|
+
# @return [Logger] the logger
|
|
11
18
|
attr_reader :logger
|
|
12
19
|
|
|
13
|
-
# @return [Proc, nil] the logger formatter
|
|
20
|
+
# @return [Proc, Logger::Formatter, nil] the logger formatter
|
|
14
21
|
attr_reader :logger_formatter
|
|
15
22
|
|
|
16
23
|
# @return [Symbol, Integer] the current log level
|
|
@@ -46,12 +53,15 @@ module Html2rss
|
|
|
46
53
|
##
|
|
47
54
|
# Sets the logger.
|
|
48
55
|
#
|
|
49
|
-
# @param logger [
|
|
50
|
-
# @return [
|
|
56
|
+
# @param logger [Logger]
|
|
57
|
+
# @return [Logger] the logger
|
|
58
|
+
# @raise [ArgumentError] if logger is not a Logger
|
|
51
59
|
def logger=(logger)
|
|
60
|
+
raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(::Logger)
|
|
61
|
+
|
|
52
62
|
@logger = logger
|
|
53
|
-
@logger.level = @log_level
|
|
54
|
-
@logger.formatter = @logger_formatter if @logger_formatter
|
|
63
|
+
@logger.level = @log_level
|
|
64
|
+
@logger.formatter = @logger_formatter if @logger_formatter
|
|
55
65
|
end
|
|
56
66
|
|
|
57
67
|
##
|
|
@@ -62,34 +72,39 @@ module Html2rss
|
|
|
62
72
|
# @raise [ArgumentError] if the log level is invalid
|
|
63
73
|
def log_level=(level)
|
|
64
74
|
@log_level = normalize_log_level(level)
|
|
65
|
-
@logger.level = @log_level
|
|
75
|
+
@logger.level = @log_level
|
|
66
76
|
end
|
|
67
77
|
|
|
68
78
|
##
|
|
69
79
|
# Sets the logger formatter.
|
|
70
80
|
#
|
|
71
|
-
# @param formatter [Proc,
|
|
72
|
-
# @return [Proc,
|
|
73
|
-
# @raise [ArgumentError] if formatter
|
|
81
|
+
# @param formatter [Proc, Logger::Formatter, nil] the new logger formatter
|
|
82
|
+
# @return [Proc, Logger::Formatter, nil] the new logger formatter
|
|
83
|
+
# @raise [ArgumentError] if formatter is not a Proc, Logger::Formatter, or nil
|
|
74
84
|
def logger_formatter=(formatter)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
85
|
+
@logger_formatter = case formatter
|
|
86
|
+
when nil, Proc, Logger::Formatter
|
|
87
|
+
formatter
|
|
88
|
+
else
|
|
89
|
+
raise ArgumentError, 'formatter must be a Proc, Logger::Formatter, or nil'
|
|
90
|
+
end
|
|
91
|
+
@logger.formatter = @logger_formatter
|
|
79
92
|
end
|
|
80
93
|
|
|
81
94
|
##
|
|
82
95
|
# Sets the global request headers.
|
|
83
96
|
#
|
|
84
|
-
# @param headers [Hash, Proc,
|
|
85
|
-
# @return [Hash, Proc,
|
|
86
|
-
# @raise [ArgumentError] if headers is not a Hash or
|
|
97
|
+
# @param headers [Hash, Proc, nil] the HTTP request headers to globally apply
|
|
98
|
+
# @return [Hash, Proc, nil] the assigned headers
|
|
99
|
+
# @raise [ArgumentError] if headers is not a Hash, Proc, or nil
|
|
87
100
|
def headers=(headers)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
101
|
+
@headers = case headers
|
|
102
|
+
when nil then nil
|
|
103
|
+
when Hash then headers.dup.freeze
|
|
104
|
+
when Proc then headers
|
|
105
|
+
else
|
|
106
|
+
raise ArgumentError, 'headers must be a Hash or Proc'
|
|
107
|
+
end
|
|
93
108
|
end
|
|
94
109
|
|
|
95
110
|
##
|
|
@@ -164,14 +179,11 @@ module Html2rss
|
|
|
164
179
|
|
|
165
180
|
def normalize_log_level(level)
|
|
166
181
|
if level.is_a?(Integer)
|
|
167
|
-
raise ArgumentError, "invalid log level: #{level}" unless
|
|
182
|
+
raise ArgumentError, "invalid log level: #{level}" unless LOG_LEVELS.value?(level)
|
|
168
183
|
|
|
169
184
|
level
|
|
170
185
|
else
|
|
171
|
-
|
|
172
|
-
raise ArgumentError, "invalid log level: #{level}" unless VALID_LOG_LEVELS.include?(sym)
|
|
173
|
-
|
|
174
|
-
Logger.const_get(sym.upcase)
|
|
186
|
+
LOG_LEVELS.fetch(level.to_s.downcase.to_sym) { raise ArgumentError, "invalid log level: #{level}" }
|
|
175
187
|
end
|
|
176
188
|
end
|
|
177
189
|
end
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'httpx'
|
|
4
4
|
require 'json'
|
|
5
5
|
|
|
6
6
|
module Html2rss
|
|
7
|
-
##
|
|
8
|
-
# CLI runtime health checks for optional scrape dependencies.
|
|
9
7
|
module Doctor
|
|
10
8
|
##
|
|
11
9
|
# Preflight checks for Botasaurus scrape-api connectivity.
|
|
12
10
|
module Botasaurus
|
|
11
|
+
Check = Data.define(:name, :ok, :detail)
|
|
12
|
+
|
|
13
13
|
##
|
|
14
14
|
# One named doctor check outcome.
|
|
15
|
-
Check
|
|
15
|
+
class Check
|
|
16
16
|
##
|
|
17
17
|
# @return [Hash{Symbol => Object}]
|
|
18
18
|
def to_h
|
|
@@ -20,9 +20,11 @@ module Html2rss
|
|
|
20
20
|
end
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
Result = Data.define(:ok, :checks, :message)
|
|
24
|
+
|
|
23
25
|
##
|
|
24
26
|
# Aggregate doctor command outcome.
|
|
25
|
-
Result
|
|
27
|
+
class Result
|
|
26
28
|
##
|
|
27
29
|
# @return [Hash{Symbol => Object}]
|
|
28
30
|
def to_h
|
|
@@ -68,15 +70,23 @@ module Html2rss
|
|
|
68
70
|
# @param base_url [String]
|
|
69
71
|
# @return [Hash{Symbol => Object}]
|
|
70
72
|
def health_check(base_url)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
response
|
|
74
|
-
|
|
73
|
+
endpoint = "#{Url.for_channel(base_url).to_s.chomp('/')}/health"
|
|
74
|
+
response = health_session.get(endpoint)
|
|
75
|
+
raise response.error if response.is_a?(HTTPX::ErrorResponse)
|
|
76
|
+
|
|
77
|
+
body = JSON.parse(response.body.to_s)
|
|
75
78
|
{ ok: response.status == 200, status: response.status, version: body['version'] }
|
|
76
79
|
rescue StandardError => error
|
|
77
80
|
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
78
81
|
end
|
|
79
82
|
|
|
83
|
+
def health_session
|
|
84
|
+
HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
|
|
85
|
+
resolver_class: :system)
|
|
86
|
+
end
|
|
87
|
+
module_function :health_session
|
|
88
|
+
private_class_method :health_session
|
|
89
|
+
|
|
80
90
|
##
|
|
81
91
|
# @param url [String]
|
|
82
92
|
# @return [Hash{Symbol => Object}]
|
data/lib/html2rss/error.rb
CHANGED
|
@@ -7,7 +7,7 @@ module Html2rss
|
|
|
7
7
|
# Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
|
|
8
8
|
class NoFeedItemsExtracted < Error
|
|
9
9
|
# Categories that append shared surface guidance to the empty-feed message.
|
|
10
|
-
SURFACE_HINT_CATEGORIES =
|
|
10
|
+
SURFACE_HINT_CATEGORIES = Set[:app_shell, :blocked_surface, :high_entropy_surface].freeze
|
|
11
11
|
|
|
12
12
|
##
|
|
13
13
|
# @param attempts [Array<Hash{Symbol => Object}>] tier attempt diagnostics
|
|
@@ -34,16 +34,10 @@ module Html2rss
|
|
|
34
34
|
private
|
|
35
35
|
|
|
36
36
|
def add_item_string_values(article, item_maker)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
}.each do |attr, value|
|
|
42
|
-
next if value.nil? || value.empty?
|
|
43
|
-
|
|
44
|
-
item_maker.public_send(:"#{attr}=", value)
|
|
45
|
-
end
|
|
46
|
-
|
|
37
|
+
item_maker.title = article.title
|
|
38
|
+
item_maker.author = article.author
|
|
39
|
+
description = ItemPresentation.description_for(article)
|
|
40
|
+
item_maker.description = description if description && !description.empty?
|
|
47
41
|
item_maker.link = article.url.to_s if article.url
|
|
48
42
|
item_maker.pubDate = article.published_at&.rfc2822
|
|
49
43
|
end
|
|
@@ -96,12 +90,13 @@ module Html2rss
|
|
|
96
90
|
@stylesheets.map { |style| Stylesheet.new(**style) }
|
|
97
91
|
end
|
|
98
92
|
|
|
99
|
-
# rubocop:disable-next Metrics/AbcSize
|
|
93
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength
|
|
100
94
|
def make_channel(maker)
|
|
101
95
|
channel_maker = maker.channel
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
96
|
+
channel_maker.language = channel.language
|
|
97
|
+
channel_maker.title = channel.title
|
|
98
|
+
channel_maker.description = channel.description
|
|
99
|
+
channel_maker.ttl = channel.ttl
|
|
105
100
|
|
|
106
101
|
channel_maker.managingEditor = channel.author if channel.author
|
|
107
102
|
channel_maker.author = channel.author if channel.author
|
|
@@ -2,25 +2,25 @@
|
|
|
2
2
|
|
|
3
3
|
`:auto` is the default request plan for feed builds (`auto_source`, `auto_json_feed`, Capture, and MCP `scrape` / `capture`). `FeedPipeline::StrategyPlan` resolves it; `FeedPipeline::AutoFallback` executes `AutoFallback::CHAIN`.
|
|
4
4
|
|
|
5
|
-
Use `:auto` when you want
|
|
5
|
+
Use `:auto` when you want `default` (HTTPX) first and a browser-backed hop only if that fetch fails or yields zero items. Pin a concrete strategy (`default`, `botasaurus`, `local_file`) when you need a single transport.
|
|
6
6
|
|
|
7
7
|
## Chain
|
|
8
8
|
|
|
9
9
|
`AutoFallback::CHAIN` is:
|
|
10
10
|
|
|
11
|
-
1. **
|
|
12
|
-
2. **Botasaurus** — attempted when
|
|
11
|
+
1. **Default (HTTPX)** — plain HTTP (faster, cheaper).
|
|
12
|
+
2. **Botasaurus** — attempted when default raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
|
|
13
13
|
|
|
14
|
-
Before escalating
|
|
14
|
+
Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
|
|
15
15
|
|
|
16
|
-
There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without
|
|
16
|
+
There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without default first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
|
|
17
17
|
|
|
18
18
|
## Surfaces
|
|
19
19
|
|
|
20
20
|
| Surface | `:auto` behavior |
|
|
21
21
|
| ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
|
|
22
|
-
| Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`
|
|
23
|
-
| MCP `inspect` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to
|
|
22
|
+
| Gem / CLI `apply`, MCP `scrape`, Capture | Full AutoFallback chain (`default` → `botasaurus`) |
|
|
23
|
+
| MCP `inspect` | Cheap diagnostic: `StrategyPlan.concrete_for_diagnostic` maps `auto` to default (pin `botasaurus` when you need browser rendering) |
|
|
24
24
|
|
|
25
25
|
## Fallback vs abort
|
|
26
26
|
|
|
@@ -28,7 +28,7 @@ These typically hop to the next chain member (among other `StandardError`s `Auto
|
|
|
28
28
|
|
|
29
29
|
- `Html2rss::RequestService::BlockedSurfaceDetected`
|
|
30
30
|
- `Html2rss::RequestService::RequestTimedOut`
|
|
31
|
-
-
|
|
31
|
+
- HTTPX connection / timeout errors
|
|
32
32
|
- Empty extraction results when a later chain member may succeed
|
|
33
33
|
|
|
34
34
|
These abort immediately (`AutoFallback::NON_FALLBACK_ERRORS`): unknown strategy, invalid URL, unsupported scheme, budget exceeded, private network denied, cross-origin follow-up denied, response too large.
|
|
@@ -8,7 +8,7 @@ module Html2rss
|
|
|
8
8
|
# Hosted by the pipeline instance: session + extract call back into FeedPipeline.
|
|
9
9
|
class AutoFallback # rubocop:disable Metrics/ClassLength -- attempt state + logging stay co-located
|
|
10
10
|
# Ordered list of concrete request strategies attempted by the :auto plan.
|
|
11
|
-
CHAIN = %i[
|
|
11
|
+
CHAIN = %i[default botasaurus].freeze
|
|
12
12
|
|
|
13
13
|
# Deterministic HTTP statuses that indicate permanent failure rather than anti-bot challenges.
|
|
14
14
|
DETERMINISTIC_HTTP_STATUSES = Set[400, 404, 410, 422, 451].freeze
|
|
@@ -45,6 +45,20 @@ module Html2rss
|
|
|
45
45
|
# @return [Html2rss::RequestService::Budget] shared budget for the feed build
|
|
46
46
|
def self.budget_for(config) = resources_for(config).budget
|
|
47
47
|
|
|
48
|
+
##
|
|
49
|
+
# Reserve enough HTTP request slots for the initial request plus predictable
|
|
50
|
+
# follow-ups.
|
|
51
|
+
#
|
|
52
|
+
# @param config [Html2rss::Config] validated feed config
|
|
53
|
+
# @return [Integer] number of reserved baseline request slots
|
|
54
|
+
def self.baseline_request_budget_for(config)
|
|
55
|
+
1 +
|
|
56
|
+
RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
|
|
57
|
+
AutoSource.request_slots_for(config.auto_source) +
|
|
58
|
+
FeedResolution.request_slots_for(config.auto_source) +
|
|
59
|
+
StrategyPlan.resolve(config.strategy).request_slots
|
|
60
|
+
end
|
|
61
|
+
|
|
48
62
|
class << self
|
|
49
63
|
private
|
|
50
64
|
|
|
@@ -53,16 +67,6 @@ module Html2rss
|
|
|
53
67
|
|
|
54
68
|
[baseline_request_budget_for(config), config.max_requests].max
|
|
55
69
|
end
|
|
56
|
-
|
|
57
|
-
# Reserve enough HTTP request slots for the initial request plus predictable
|
|
58
|
-
# follow-ups.
|
|
59
|
-
def baseline_request_budget_for(config)
|
|
60
|
-
1 +
|
|
61
|
-
RequestSession::Pager.request_slots_for(config.selectors&.dig(:items, :pagination)) +
|
|
62
|
-
AutoSource.request_slots_for(config.auto_source) +
|
|
63
|
-
FeedResolution.request_slots_for(config.auto_source) +
|
|
64
|
-
StrategyPlan.resolve(config.strategy).request_slots
|
|
65
|
-
end
|
|
66
70
|
end
|
|
67
71
|
end
|
|
68
72
|
end
|
|
@@ -8,16 +8,20 @@ module Html2rss
|
|
|
8
8
|
# {RequestService} executes concrete adapters only. +:auto+ is resolved here before
|
|
9
9
|
# any session or adapter call — plan vs transport locality.
|
|
10
10
|
class StrategyPlan
|
|
11
|
+
Auto = Data.define
|
|
12
|
+
|
|
11
13
|
# Feed-level auto plan: try {AutoFallback::CHAIN} until items are extracted.
|
|
12
|
-
Auto
|
|
14
|
+
class Auto
|
|
13
15
|
# @return [Integer]
|
|
14
16
|
def request_slots
|
|
15
17
|
[AutoFallback::CHAIN.size - 1, 0].max
|
|
16
18
|
end
|
|
17
19
|
end
|
|
18
20
|
|
|
21
|
+
Concrete = Data.define(:strategy)
|
|
22
|
+
|
|
19
23
|
# Concrete transport strategy name for {RequestService}.
|
|
20
|
-
Concrete
|
|
24
|
+
class Concrete
|
|
21
25
|
# @return [Integer]
|
|
22
26
|
def request_slots = 0
|
|
23
27
|
end
|
|
@@ -42,7 +46,7 @@ module Html2rss
|
|
|
42
46
|
end
|
|
43
47
|
|
|
44
48
|
##
|
|
45
|
-
# Cheap single-request diagnostic default: +:auto+ → +:
|
|
49
|
+
# Cheap single-request diagnostic default: +:auto+ → +:default+.
|
|
46
50
|
# Does not run {AutoFallback}. Prefer {resolve} for scrape/capture feeds.
|
|
47
51
|
#
|
|
48
52
|
# @param name [Symbol, String, nil] plan name (+nil+ → +:auto+)
|
|
@@ -99,11 +99,8 @@ module Html2rss
|
|
|
99
99
|
|
|
100
100
|
def segment_urls_for(sst, strategy, link_resolver)
|
|
101
101
|
segmenter = AutoSource::Segmenter.new(sst, base_url: entry_url, strategy:, link_resolver:)
|
|
102
|
-
primary = AutoSource::Segmenter::PrimaryLink.new(segmenter)
|
|
103
|
-
|
|
104
102
|
segmenter.call.filter_map do |segment|
|
|
105
|
-
|
|
106
|
-
absolute_same_origin(link&.attrs&.href)
|
|
103
|
+
absolute_same_origin(segment.primary_link&.attrs&.href)
|
|
107
104
|
end.uniq
|
|
108
105
|
end
|
|
109
106
|
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
module FeedResolution
|
|
5
|
+
Diag = Data.define(:applied, :probe_count, :reason, :winner_score)
|
|
6
|
+
|
|
5
7
|
##
|
|
6
8
|
# Wire-safe diagnostics for one entry-resolution attempt (Status / Marshal edge).
|
|
7
|
-
Diag
|
|
9
|
+
class Diag
|
|
8
10
|
##
|
|
9
11
|
# @param result [Html2rss::FeedResolution::Result]
|
|
10
12
|
# @return [Diag]
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
module FeedResolution
|
|
5
|
+
Options = Data.define(:enabled, :max_probes)
|
|
6
|
+
|
|
5
7
|
##
|
|
6
8
|
# Typed +auto_source.entry_resolution+ options (one expansion from config Hash).
|
|
7
|
-
Options
|
|
9
|
+
class Options
|
|
8
10
|
##
|
|
9
11
|
# @param auto_source [Hash, nil]
|
|
10
12
|
# @return [Options]
|
|
@@ -13,13 +13,13 @@ module Html2rss
|
|
|
13
13
|
##
|
|
14
14
|
# @param config [Html2rss::Config]
|
|
15
15
|
# @param articles [Array<Html2rss::Article>] typed extract results (count derived)
|
|
16
|
-
# @param surface_category [Html2rss::SurfaceCategory, Symbol
|
|
16
|
+
# @param surface_category [Html2rss::SurfaceCategory, Symbol]
|
|
17
17
|
# @return [Boolean]
|
|
18
18
|
def resolve?(config:, articles:, surface_category:)
|
|
19
19
|
raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
|
|
20
20
|
return false unless eligible_config?(config)
|
|
21
21
|
|
|
22
|
-
category = SurfaceCategory
|
|
22
|
+
category = SurfaceCategory[surface_category]
|
|
23
23
|
return false if category.blocked?
|
|
24
24
|
|
|
25
25
|
articles.size < ARTICLE_FLOOR || category.weak? || native_feed_majority?(articles)
|
|
@@ -10,7 +10,7 @@ module Html2rss
|
|
|
10
10
|
# @param time_zone [String] channel time zone for naive values
|
|
11
11
|
# @return [DateTime, nil]
|
|
12
12
|
def self.call(article_tag, leftover_lines: [], time_zone: 'UTC')
|
|
13
|
-
datetimes = article_tag.
|
|
13
|
+
datetimes = article_tag.css('[datetime]').map { |tag| tag['datetime'] }
|
|
14
14
|
ArticleRules::Date.earliest(datetimes, leftover_lines:, time_zone:)
|
|
15
15
|
end
|
|
16
16
|
end
|
|
@@ -124,8 +124,7 @@ module Html2rss
|
|
|
124
124
|
end
|
|
125
125
|
|
|
126
126
|
def leftover_exclude_nodes_for(node)
|
|
127
|
-
|
|
128
|
-
[heading, selected_anchor, kicker_node, *times].compact
|
|
127
|
+
[heading, selected_anchor, kicker_node, *node.css('time')].compact
|
|
129
128
|
end
|
|
130
129
|
|
|
131
130
|
def heading_or_anchor_item?
|
|
@@ -156,8 +155,7 @@ module Html2rss
|
|
|
156
155
|
end
|
|
157
156
|
|
|
158
157
|
def immediate_card_parent
|
|
159
|
-
parent
|
|
160
|
-
Navigator.parent_until_condition(parent, method(:usable_walk_parent?))
|
|
158
|
+
Navigator.parent_until_condition(article_tag.parent, ->(node) { usable_walk_parent?(node) })
|
|
161
159
|
end
|
|
162
160
|
|
|
163
161
|
def usable_walk_parent?(node)
|