html2rss 0.32.4 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/lib/html2rss/article/deduplicator.rb +5 -2
- data/lib/html2rss/article/enclosure.rb +185 -25
- data/lib/html2rss/article/title.rb +203 -0
- data/lib/html2rss/article.rb +79 -13
- data/lib/html2rss/auto_source/README.md +16 -8
- data/lib/html2rss/auto_source/cleanup.rb +127 -53
- data/lib/html2rss/auto_source/corroboration.rb +167 -0
- data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
- data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
- data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
- data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
- data/lib/html2rss/auto_source/page_links.rb +164 -0
- data/lib/html2rss/auto_source/scraper/html.rb +11 -45
- data/lib/html2rss/auto_source/scraper/json_document.rb +3 -6
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +202 -125
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +27 -38
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +27 -35
- data/lib/html2rss/auto_source/scraper/json_state.rb +4 -9
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +27 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +233 -248
- data/lib/html2rss/auto_source/scraper/microformats2.rb +10 -4
- data/lib/html2rss/auto_source/scraper/native_feed.rb +28 -18
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +140 -22
- data/lib/html2rss/auto_source/scraper/schema.rb +52 -8
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -20
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +7 -14
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +7 -7
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +2 -4
- data/lib/html2rss/auto_source/scraper.rb +10 -13
- data/lib/html2rss/auto_source/segment.rb +19 -5
- data/lib/html2rss/auto_source/segmenter/cluster.rb +18 -22
- data/lib/html2rss/auto_source/segmenter/list.rb +35 -59
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
- data/lib/html2rss/auto_source/segmenter/semantic.rb +23 -19
- data/lib/html2rss/auto_source/segmenter.rb +4 -8
- data/lib/html2rss/auto_source.rb +81 -26
- data/lib/html2rss/batch.rb +24 -32
- data/lib/html2rss/capture/README.md +6 -3
- data/lib/html2rss/capture.rb +51 -59
- data/lib/html2rss/channel.rb +30 -12
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +55 -38
- data/lib/html2rss/config/auto_source_contract.rb +52 -45
- data/lib/html2rss/config/issue_mapper.rb +4 -1
- data/lib/html2rss/config/request_controls.rb +16 -5
- data/lib/html2rss/config/schema.rb +23 -15
- data/lib/html2rss/config/selectors_validator.rb +3 -2
- data/lib/html2rss/config/validation_issue.rb +4 -1
- data/lib/html2rss/config/validator.rb +14 -3
- data/lib/html2rss/config.rb +54 -28
- data/lib/html2rss/contract_error.rb +9 -0
- data/lib/html2rss/defaults.rb +25 -38
- data/lib/html2rss/doctor/botasaurus.rb +18 -22
- data/lib/html2rss/error.rb +17 -3
- data/lib/html2rss/error_kind.rb +39 -0
- data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
- data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
- data/lib/html2rss/feed_builder/json_feed.rb +2 -2
- data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
- data/lib/html2rss/feed_builder/rss.rb +50 -7
- data/lib/html2rss/feed_builder.rb +2 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +16 -34
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
- data/lib/html2rss/feed_pipeline.rb +40 -27
- data/lib/html2rss/feed_resolution/README.md +1 -1
- data/lib/html2rss/feed_resolution/candidate_generator.rb +6 -8
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +6 -4
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +16 -21
- data/lib/html2rss/feed_result.rb +11 -6
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +7 -10
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -33
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +7 -10
- data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
- data/lib/html2rss/html/article_extractor.rb +19 -47
- data/lib/html2rss/html/article_rules/category.rb +3 -3
- data/lib/html2rss/html/article_rules/description.rb +6 -15
- data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
- data/lib/html2rss/html/article_rules/image.rb +30 -16
- data/lib/html2rss/html/feed_link.rb +7 -9
- data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
- data/lib/html2rss/html/navigator.rb +39 -9
- data/lib/html2rss/html/probe.rb +55 -51
- data/lib/html2rss/html/rendering/description_builder.rb +46 -32
- data/lib/html2rss/html/rendering.rb +2 -9
- data/lib/html2rss/html/sst_article_extractor.rb +55 -145
- data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
- data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
- data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
- data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
- data/lib/html2rss/html/title_pieces.rb +169 -0
- data/lib/html2rss/html/visibility.rb +56 -0
- data/lib/html2rss/html.rb +2 -0
- data/lib/html2rss/link_destination/destination_facts.rb +17 -1
- data/lib/html2rss/link_destination/noise_policy.rb +11 -10
- data/lib/html2rss/link_destination/path_classifier.rb +111 -42
- data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
- data/lib/html2rss/link_destination/sponsorship.rb +184 -0
- data/lib/html2rss/link_destination/text_classifier.rb +104 -8
- data/lib/html2rss/mcp/README.md +13 -1
- data/lib/html2rss/mcp/config_argument.rb +14 -4
- data/lib/html2rss/mcp/contract.rb +20 -9
- data/lib/html2rss/mcp/outcome.rb +18 -17
- data/lib/html2rss/mcp/runtime.rb +53 -23
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/mcp/server.rb +13 -4
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/README.md +1 -1
- data/lib/html2rss/page_recon/diagnostics.rb +9 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +19 -23
- data/lib/html2rss/request_service/blocked_surface.rb +5 -8
- data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
- data/lib/html2rss/request_service/botasaurus_strategy.rb +44 -12
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +16 -17
- data/lib/html2rss/request_service/cookie_jar.rb +89 -0
- data/lib/html2rss/request_service/cookie_store.rb +79 -0
- data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
- data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +2 -3
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +55 -36
- data/lib/html2rss/request_service/replay_breaker.rb +56 -0
- data/lib/html2rss/request_service/response.rb +33 -19
- data/lib/html2rss/request_service/response_cache.rb +109 -0
- data/lib/html2rss/request_service/response_caching.rb +95 -0
- data/lib/html2rss/request_service/strategy.rb +5 -11
- data/lib/html2rss/request_service.rb +71 -12
- data/lib/html2rss/request_session/pager/base.rb +3 -10
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/request_session/pager.rb +1 -1
- data/lib/html2rss/request_session.rb +6 -2
- data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
- data/lib/html2rss/scoring/container_assessor.rb +12 -13
- data/lib/html2rss/scoring/engine.rb +10 -38
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/scoring/observation.rb +2 -2
- data/lib/html2rss/selector_candidates.rb +19 -8
- data/lib/html2rss/selectors/README.md +1 -1
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/option_spec.rb +4 -1
- data/lib/html2rss/selectors/post_processors/base.rb +0 -8
- data/lib/html2rss/selectors/post_processors/gsub.rb +9 -10
- data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +5 -6
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
- data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +67 -30
- data/lib/html2rss/sst/attrs.rb +2 -2
- data/lib/html2rss/sst/document.rb +3 -3
- data/lib/html2rss/sst/index.rb +30 -11
- data/lib/html2rss/sst/node.rb +42 -29
- data/lib/html2rss/sst/normalizer.rb +47 -34
- data/lib/html2rss/sst/tags.rb +113 -2
- data/lib/html2rss/sst/text.rb +94 -58
- data/lib/html2rss/sst.rb +2 -0
- data/lib/html2rss/status.rb +27 -31
- data/lib/html2rss/surface_category.rb +1 -1
- data/lib/html2rss/syndication/README.md +12 -6
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/channel_facts.rb +87 -0
- data/lib/html2rss/syndication/discovery.rb +171 -187
- data/lib/html2rss/syndication/feed.rb +20 -0
- data/lib/html2rss/syndication/item_media.rb +103 -0
- data/lib/html2rss/syndication/parser.rb +117 -116
- data/lib/html2rss/test/enhance_audit.rb +15 -17
- data/lib/html2rss/test/item_sample.rb +43 -0
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +66 -57
- data/lib/html2rss/url.rb +163 -62
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss/word_count.rb +53 -0
- data/lib/html2rss.rb +19 -9
- data/schema/html2rss-config.schema.json +56 -41
- metadata +33 -4
- data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
- data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
data/lib/html2rss/config.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Html2rss
|
|
|
11
11
|
#
|
|
12
12
|
# Configuration is validated during initialization.
|
|
13
13
|
class Config # rubocop:disable Metrics/ClassLength
|
|
14
|
-
# Raised when a configuration
|
|
14
|
+
# Raised when a configuration cannot be loaded (YAML shape, file, feed name) or fails runtime validation.
|
|
15
15
|
class InvalidConfig < Html2rss::Error; end
|
|
16
16
|
|
|
17
17
|
class << self
|
|
@@ -25,7 +25,7 @@ module Html2rss
|
|
|
25
25
|
# @return [Array(Hash, Html2rss::Config::ValidationReport)]
|
|
26
26
|
def resolve_and_validate(config_input, feed_name: nil, params: {})
|
|
27
27
|
raw = resolve_raw_hash(config_input, feed_name)
|
|
28
|
-
rescue
|
|
28
|
+
rescue InvalidConfig => error
|
|
29
29
|
[{}, IssueMapper.parse_failure(error.message)]
|
|
30
30
|
else
|
|
31
31
|
working = HashUtil.deep_dup(raw)
|
|
@@ -66,14 +66,16 @@ module Html2rss
|
|
|
66
66
|
#
|
|
67
67
|
# @param string [String] YAML document
|
|
68
68
|
# @return [Hash{Symbol => Object}] configuration hash
|
|
69
|
-
# @raise [
|
|
69
|
+
# @raise [InvalidConfig] if +string+ is not a String or does not deserialize to a Hash
|
|
70
70
|
def from_yaml(string)
|
|
71
|
-
raise
|
|
71
|
+
raise InvalidConfig, 'YAML must be a String' unless string.is_a?(String)
|
|
72
72
|
|
|
73
73
|
parsed = YAML.safe_load(string)
|
|
74
|
-
raise
|
|
74
|
+
raise InvalidConfig, 'YAML must deserialize to a Hash' unless parsed.is_a?(Hash)
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
symbolize_config(parsed)
|
|
77
|
+
rescue Psych::Exception => error
|
|
78
|
+
raise InvalidConfig, "YAML could not be parsed: #{error.message}"
|
|
77
79
|
end
|
|
78
80
|
|
|
79
81
|
##
|
|
@@ -85,25 +87,16 @@ module Html2rss
|
|
|
85
87
|
# @param feed_name [String, nil] the feed name when using multiple feeds.
|
|
86
88
|
# @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
|
|
87
89
|
# @return [Hash{Symbol => Object}] the configuration hash.
|
|
88
|
-
# @raise [
|
|
89
|
-
# rubocop:disable-next Metrics/MethodLength
|
|
90
|
+
# @raise [InvalidConfig] if the file doesn't exist or feed is not found.
|
|
90
91
|
def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
|
|
91
|
-
raise
|
|
92
|
-
raise
|
|
92
|
+
raise InvalidConfig, "File '#{file}' does not exist" unless File.exist?(file)
|
|
93
|
+
raise InvalidConfig, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
|
|
93
94
|
|
|
94
|
-
yaml =
|
|
95
|
+
yaml = read_yaml_mapping(file)
|
|
95
96
|
|
|
96
97
|
return yaml unless yaml.key?(multiple_feeds_key)
|
|
97
98
|
|
|
98
|
-
|
|
99
|
-
available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
|
|
100
|
-
raise ArgumentError,
|
|
101
|
-
"Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
|
|
105
|
-
raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
|
|
106
|
-
|
|
99
|
+
config = select_feed(file, feed_name, yaml.fetch(multiple_feeds_key), multiple_feeds_key)
|
|
107
100
|
MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
|
|
108
101
|
end
|
|
109
102
|
|
|
@@ -126,17 +119,15 @@ module Html2rss
|
|
|
126
119
|
# @param request_controls [Html2rss::Config::RequestControls, nil] explicit request controls to write
|
|
127
120
|
# @param limit [Integer, nil] max articles to keep in the auto-sourced feed
|
|
128
121
|
# @param keep_different_domain [Boolean, nil] when set, overrides cleanup.keep_different_domain
|
|
122
|
+
# @param keep_sponsored [Boolean, nil] when set, overrides cleanup.keep_sponsored
|
|
129
123
|
# @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
|
|
130
|
-
# rubocop:disable-next Metrics/MethodLength --
|
|
124
|
+
# rubocop:disable-next Metrics/MethodLength, Metrics/ParameterLists -- cleanup overrides stay co-located
|
|
131
125
|
def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil,
|
|
132
|
-
keep_different_domain: nil)
|
|
126
|
+
keep_different_domain: nil, keep_sponsored: nil)
|
|
133
127
|
auto_source = AutoSource::DEFAULT_CONFIG
|
|
134
|
-
auto_source = auto_source.merge(limit:) unless limit.nil?
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
cleanup: auto_source[:cleanup].merge(keep_different_domain:)
|
|
138
|
-
)
|
|
139
|
-
end
|
|
128
|
+
auto_source = auto_source.merge(limit: Option.integer(:limit, limit, min: 1)) unless limit.nil?
|
|
129
|
+
cleanup = { keep_different_domain:, keep_sponsored: }.compact
|
|
130
|
+
auto_source = auto_source.merge(cleanup: auto_source[:cleanup].merge(cleanup)) unless cleanup.empty?
|
|
140
131
|
|
|
141
132
|
config = {
|
|
142
133
|
channel: default_config[:channel].merge(url:),
|
|
@@ -228,6 +219,41 @@ module Html2rss
|
|
|
228
219
|
end
|
|
229
220
|
end
|
|
230
221
|
private :resolve_raw_hash
|
|
222
|
+
|
|
223
|
+
def read_yaml_mapping(file)
|
|
224
|
+
yaml = YAML.safe_load_file(file)
|
|
225
|
+
return symbolize_config(yaml) if yaml.is_a?(Hash)
|
|
226
|
+
|
|
227
|
+
raise InvalidConfig, "File '#{file}' must contain a YAML mapping (a Hash)"
|
|
228
|
+
rescue Psych::Exception => error
|
|
229
|
+
raise InvalidConfig, "File '#{file}' could not be parsed: #{error.message}"
|
|
230
|
+
end
|
|
231
|
+
private :read_yaml_mapping
|
|
232
|
+
|
|
233
|
+
# @return [Hash{Symbol => Object}] the named feed's config from the +feeds+ mapping
|
|
234
|
+
def select_feed(file, feed_name, feeds, key)
|
|
235
|
+
unless feeds.is_a?(Hash)
|
|
236
|
+
raise InvalidConfig, "File '#{file}': `#{key}` must be a mapping of feed names to configs"
|
|
237
|
+
end
|
|
238
|
+
unless feed_name
|
|
239
|
+
raise InvalidConfig, "Feed name is required under `#{key}`. Available feeds: #{feeds.keys.join(', ')}"
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
config = feeds[feed_name.to_sym]
|
|
243
|
+
raise InvalidConfig, "Feed '#{feed_name}' not found under `#{key}` key." unless config
|
|
244
|
+
raise InvalidConfig, "File '#{file}': feed '#{feed_name}' must be a mapping" unless config.is_a?(Hash)
|
|
245
|
+
|
|
246
|
+
config
|
|
247
|
+
end
|
|
248
|
+
private :select_feed
|
|
249
|
+
|
|
250
|
+
# Loaded config keys that cannot be symbolized (non-string, non-symbol) are a config-load failure.
|
|
251
|
+
def symbolize_config(parsed)
|
|
252
|
+
HashUtil.deep_symbolize_keys(parsed, context: 'config')
|
|
253
|
+
rescue InvalidOption => error
|
|
254
|
+
raise InvalidConfig, error.message
|
|
255
|
+
end
|
|
256
|
+
private :symbolize_config
|
|
231
257
|
end
|
|
232
258
|
|
|
233
259
|
##
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
##
|
|
5
|
+
# Base for programmer errors: a value object built from wrong parts (each owner's +InvalidError+) or a
|
|
6
|
+
# component breaking its contract ({AutoSource::ScraperContractError}). It is an {Error}, but no
|
|
7
|
+
# operational rescue may swallow it: every broad +rescue Html2rss::Error+ in +lib/+ re-raises it first.
|
|
8
|
+
class ContractError < Error; end
|
|
9
|
+
end
|
data/lib/html2rss/defaults.rb
CHANGED
|
@@ -55,9 +55,9 @@ module Html2rss
|
|
|
55
55
|
#
|
|
56
56
|
# @param logger [Logger]
|
|
57
57
|
# @return [Logger] the logger
|
|
58
|
-
# @raise [
|
|
58
|
+
# @raise [InvalidOption] if logger is not a Logger
|
|
59
59
|
def logger=(logger)
|
|
60
|
-
raise
|
|
60
|
+
raise InvalidOption, 'logger must be a Logger' unless logger.is_a?(Logger)
|
|
61
61
|
|
|
62
62
|
@logger = logger
|
|
63
63
|
@logger.level = @log_level
|
|
@@ -69,7 +69,7 @@ module Html2rss
|
|
|
69
69
|
#
|
|
70
70
|
# @param level [Symbol, String, Integer] the new log level
|
|
71
71
|
# @return [Integer] the normalized log level
|
|
72
|
-
# @raise [
|
|
72
|
+
# @raise [InvalidOption] if the log level is invalid
|
|
73
73
|
def log_level=(level)
|
|
74
74
|
@log_level = normalize_log_level(level)
|
|
75
75
|
@logger.level = @log_level
|
|
@@ -80,13 +80,13 @@ module Html2rss
|
|
|
80
80
|
#
|
|
81
81
|
# @param formatter [Proc, Logger::Formatter, nil] the new logger formatter
|
|
82
82
|
# @return [Proc, Logger::Formatter, nil] the new logger formatter
|
|
83
|
-
# @raise [
|
|
83
|
+
# @raise [InvalidOption] if formatter is not a Proc, Logger::Formatter, or nil
|
|
84
84
|
def logger_formatter=(formatter)
|
|
85
85
|
@logger_formatter = case formatter
|
|
86
86
|
when nil, Proc, Logger::Formatter
|
|
87
87
|
formatter
|
|
88
88
|
else
|
|
89
|
-
raise
|
|
89
|
+
raise InvalidOption, 'formatter must be a Proc, Logger::Formatter, or nil'
|
|
90
90
|
end
|
|
91
91
|
@logger.formatter = @logger_formatter
|
|
92
92
|
end
|
|
@@ -96,14 +96,14 @@ module Html2rss
|
|
|
96
96
|
#
|
|
97
97
|
# @param headers [Hash, Proc, nil] the HTTP request headers to globally apply
|
|
98
98
|
# @return [Hash, Proc, nil] the assigned headers
|
|
99
|
-
# @raise [
|
|
99
|
+
# @raise [InvalidOption] if headers is not a Hash, Proc, or nil
|
|
100
100
|
def headers=(headers)
|
|
101
101
|
@headers = case headers
|
|
102
102
|
when nil then nil
|
|
103
103
|
when Hash then headers.dup.freeze
|
|
104
104
|
when Proc then headers
|
|
105
105
|
else
|
|
106
|
-
raise
|
|
106
|
+
raise InvalidOption, 'headers must be a Hash or Proc'
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
109
|
|
|
@@ -114,20 +114,9 @@ module Html2rss
|
|
|
114
114
|
#
|
|
115
115
|
# @param strategy [Symbol, String, nil] the strategy plan name
|
|
116
116
|
# @return [Symbol, nil] the normalized strategy plan name
|
|
117
|
-
# @raise [
|
|
117
|
+
# @raise [InvalidOption] if the strategy plan is unknown
|
|
118
118
|
def default_strategy=(strategy)
|
|
119
|
-
|
|
120
|
-
@default_strategy = nil
|
|
121
|
-
else
|
|
122
|
-
unless strategy.is_a?(Symbol) || strategy.is_a?(String)
|
|
123
|
-
raise ArgumentError, 'strategy must be a Symbol or String'
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
normalized = strategy.to_sym
|
|
127
|
-
raise ArgumentError, "unknown strategy: #{strategy}" unless FeedPipeline::StrategyPlan.valid?(normalized)
|
|
128
|
-
|
|
129
|
-
@default_strategy = normalized
|
|
130
|
-
end
|
|
119
|
+
@default_strategy = (Option.strategy(strategy) unless strategy.nil?)
|
|
131
120
|
end
|
|
132
121
|
|
|
133
122
|
##
|
|
@@ -135,18 +124,9 @@ module Html2rss
|
|
|
135
124
|
#
|
|
136
125
|
# @param ttl [Integer, String, nil] the minimum TTL
|
|
137
126
|
# @return [Integer, nil] the normalized minimum TTL
|
|
138
|
-
# @raise [
|
|
127
|
+
# @raise [InvalidOption] if ttl is not a positive integer
|
|
139
128
|
def min_ttl=(ttl)
|
|
140
|
-
|
|
141
|
-
@min_ttl = nil
|
|
142
|
-
else
|
|
143
|
-
val = Integer(ttl)
|
|
144
|
-
raise ArgumentError unless val.positive?
|
|
145
|
-
|
|
146
|
-
@min_ttl = val
|
|
147
|
-
end
|
|
148
|
-
rescue ArgumentError, TypeError
|
|
149
|
-
raise ArgumentError, "min_ttl must be a positive integer, got #{ttl.inspect}"
|
|
129
|
+
@min_ttl = (positive_ttl(ttl) unless ttl.nil?)
|
|
150
130
|
end
|
|
151
131
|
|
|
152
132
|
##
|
|
@@ -154,10 +134,10 @@ module Html2rss
|
|
|
154
134
|
#
|
|
155
135
|
# @param stylesheets [Array<Hash>] the XML stylesheet processing instructions to include in the generated feed
|
|
156
136
|
# @return [Array<Hash>] the assigned stylesheets
|
|
157
|
-
# @raise [
|
|
137
|
+
# @raise [InvalidOption] if stylesheets is not an Array of hashes
|
|
158
138
|
def stylesheets=(stylesheets)
|
|
159
|
-
raise
|
|
160
|
-
raise
|
|
139
|
+
raise InvalidOption, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
|
|
140
|
+
raise InvalidOption, 'stylesheets must be an Array of Hashes' unless stylesheets.all?(Hash)
|
|
161
141
|
|
|
162
142
|
@stylesheets = stylesheets.map { |h| h.dup.freeze }.freeze
|
|
163
143
|
end
|
|
@@ -171,19 +151,26 @@ module Html2rss
|
|
|
171
151
|
# @return [void]
|
|
172
152
|
def initialize_copy(other)
|
|
173
153
|
super
|
|
174
|
-
@headers =
|
|
175
|
-
@stylesheets =
|
|
154
|
+
@headers = other.headers.dup if other.headers.is_a?(Hash)
|
|
155
|
+
@stylesheets = other.stylesheets.map(&:dup)
|
|
176
156
|
end
|
|
177
157
|
|
|
178
158
|
private
|
|
179
159
|
|
|
160
|
+
def positive_ttl(ttl)
|
|
161
|
+
val = Integer(ttl, exception: false)
|
|
162
|
+
return val if val&.positive?
|
|
163
|
+
|
|
164
|
+
raise InvalidOption, "min_ttl must be a positive integer, got #{ttl.inspect}"
|
|
165
|
+
end
|
|
166
|
+
|
|
180
167
|
def normalize_log_level(level)
|
|
181
168
|
if level.is_a?(Integer)
|
|
182
|
-
raise
|
|
169
|
+
raise InvalidOption, "invalid log level: #{level}" unless LOG_LEVELS.value?(level)
|
|
183
170
|
|
|
184
171
|
level
|
|
185
172
|
else
|
|
186
|
-
LOG_LEVELS.fetch(level.to_s.downcase.to_sym) { raise
|
|
173
|
+
LOG_LEVELS.fetch(level.to_s.downcase.to_sym) { raise InvalidOption, "invalid log level: #{level}" }
|
|
187
174
|
end
|
|
188
175
|
end
|
|
189
176
|
end
|
|
@@ -32,15 +32,13 @@ module Html2rss
|
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
module_function
|
|
36
|
-
|
|
37
35
|
ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
|
|
38
36
|
private_constant :ENV_VAR
|
|
39
37
|
|
|
40
38
|
##
|
|
41
39
|
# @param sample_url [String, nil] optional URL for a sample scrape smoke test
|
|
42
40
|
# @return [Result]
|
|
43
|
-
def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
|
|
41
|
+
def self.call(sample_url: nil) # rubocop:disable Metrics/AbcSize
|
|
44
42
|
env = env_check
|
|
45
43
|
checks = [env_check_record(env)]
|
|
46
44
|
return missing_env_failure(checks) unless checks.first.ok
|
|
@@ -57,7 +55,7 @@ module Html2rss
|
|
|
57
55
|
|
|
58
56
|
##
|
|
59
57
|
# @return [Hash{Symbol => Object}]
|
|
60
|
-
def env_check
|
|
58
|
+
def self.env_check
|
|
61
59
|
base = ENV.fetch(ENV_VAR, '').strip
|
|
62
60
|
{
|
|
63
61
|
configured: MCP::Runtime.botasaurus_configured?,
|
|
@@ -69,71 +67,69 @@ module Html2rss
|
|
|
69
67
|
##
|
|
70
68
|
# @param base_url [String]
|
|
71
69
|
# @return [Hash{Symbol => Object}]
|
|
72
|
-
def health_check(base_url)
|
|
73
|
-
|
|
74
|
-
response = health_session.get(endpoint)
|
|
70
|
+
def self.health_check(base_url)
|
|
71
|
+
response = health_session.get("#{Url.for_channel(base_url).to_s.chomp('/')}/health")
|
|
75
72
|
raise response.error if response.is_a?(HTTPX::ErrorResponse)
|
|
76
73
|
|
|
77
74
|
body = JSON.parse(response.body.to_s)
|
|
78
|
-
|
|
75
|
+
status = response.status
|
|
76
|
+
{ ok: status == 200, status:, version: body['version'] }
|
|
77
|
+
rescue Html2rss::ContractError
|
|
78
|
+
raise
|
|
79
79
|
rescue StandardError => error
|
|
80
80
|
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
-
def health_session
|
|
83
|
+
def self.health_session
|
|
84
84
|
HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
|
|
85
85
|
resolver_class: :system)
|
|
86
86
|
end
|
|
87
|
-
module_function :health_session
|
|
88
87
|
private_class_method :health_session
|
|
89
88
|
|
|
90
89
|
##
|
|
91
90
|
# @param url [String]
|
|
92
91
|
# @return [Hash{Symbol => Object}]
|
|
93
|
-
def sample_scrape(url)
|
|
92
|
+
def self.sample_scrape(url)
|
|
94
93
|
wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
|
|
95
94
|
{
|
|
96
95
|
ok: wire[:status].to_i.between?(200, 399),
|
|
97
96
|
status: wire[:status],
|
|
98
97
|
articles_count: wire[:articles_count]
|
|
99
98
|
}
|
|
100
|
-
rescue
|
|
99
|
+
rescue Html2rss::ContractError
|
|
100
|
+
raise
|
|
101
|
+
rescue Html2rss::Error => error
|
|
101
102
|
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
102
103
|
end
|
|
103
104
|
|
|
104
|
-
def env_check_record(env = env_check)
|
|
105
|
+
def self.env_check_record(env = env_check)
|
|
105
106
|
Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
|
|
106
107
|
end
|
|
107
|
-
module_function :env_check_record
|
|
108
108
|
private_class_method :env_check_record
|
|
109
109
|
|
|
110
|
-
def health_check_record(base_url)
|
|
110
|
+
def self.health_check_record(base_url)
|
|
111
111
|
return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
|
|
112
112
|
|
|
113
113
|
health = health_check(base_url)
|
|
114
114
|
Check.new(name: :health, ok: health[:ok], detail: health)
|
|
115
115
|
end
|
|
116
|
-
module_function :health_check_record
|
|
117
116
|
private_class_method :health_check_record
|
|
118
117
|
|
|
119
|
-
def sample_scrape_record(url)
|
|
118
|
+
def self.sample_scrape_record(url)
|
|
120
119
|
sample = sample_scrape(url)
|
|
121
120
|
Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
|
|
122
121
|
end
|
|
123
|
-
module_function :sample_scrape_record
|
|
124
122
|
private_class_method :sample_scrape_record
|
|
125
123
|
|
|
126
|
-
def missing_env_failure(checks)
|
|
124
|
+
def self.missing_env_failure(checks)
|
|
127
125
|
Log.info("doctor botasaurus: #{ENV_VAR} unset")
|
|
128
126
|
failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
|
|
129
127
|
end
|
|
130
|
-
module_function :missing_env_failure
|
|
131
128
|
private_class_method :missing_env_failure
|
|
132
129
|
|
|
133
|
-
def failure(checks, message)
|
|
130
|
+
def self.failure(checks, message)
|
|
134
131
|
Result.new(ok: false, checks:, message:)
|
|
135
132
|
end
|
|
136
|
-
module_function :failure
|
|
137
133
|
private_class_method :failure
|
|
138
134
|
end
|
|
139
135
|
end
|
data/lib/html2rss/error.rb
CHANGED
|
@@ -39,11 +39,21 @@ module Html2rss
|
|
|
39
39
|
# @return [Hash{String => Object}, nil] allowlisted upstream transport telemetry when attached
|
|
40
40
|
attr_accessor :transport_meta
|
|
41
41
|
|
|
42
|
+
##
|
|
43
|
+
# Stable failure classification, overridden per error class.
|
|
44
|
+
#
|
|
45
|
+
# @return [Symbol] one of +:timeout+, +:blocked+, +:connection+, +:invalid_url+, +:budget+, +:other+
|
|
46
|
+
def kind = :other
|
|
47
|
+
|
|
48
|
+
##
|
|
49
|
+
# @return [Boolean] whether repeating the same request may succeed (derived from {#kind})
|
|
50
|
+
def retryable? = ErrorKind[kind].retryable?
|
|
51
|
+
|
|
42
52
|
# @return [String, nil] scrape-api timeout stage when known (overridden by RequestTimedOut)
|
|
43
|
-
def timeout_phase
|
|
53
|
+
def timeout_phase; end
|
|
44
54
|
|
|
45
55
|
# @return [Symbol, nil] surface classification when known (overridden by NoFeedItemsExtracted)
|
|
46
|
-
def surface_category
|
|
56
|
+
def surface_category; end
|
|
47
57
|
|
|
48
58
|
##
|
|
49
59
|
# Sparse telemetry hash for observability consumers.
|
|
@@ -59,6 +69,9 @@ module Html2rss
|
|
|
59
69
|
end
|
|
60
70
|
end
|
|
61
71
|
|
|
72
|
+
# Raised when a caller-supplied option value is malformed or out of range.
|
|
73
|
+
class InvalidOption < Error; end
|
|
74
|
+
|
|
62
75
|
# Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
|
|
63
76
|
class NoFeedItemsExtracted < Error
|
|
64
77
|
# Categories that append shared surface guidance to the empty-feed message.
|
|
@@ -87,7 +100,8 @@ module Html2rss
|
|
|
87
100
|
|
|
88
101
|
def base_message
|
|
89
102
|
summaries = attempts.map do |attempt|
|
|
90
|
-
|
|
103
|
+
count = attempt[:items_count]
|
|
104
|
+
details = count.nil? ? "#{attempt[:error_class]} error" : "#{count} items"
|
|
91
105
|
"#{attempt[:strategy]} (#{details})"
|
|
92
106
|
end.join(', ')
|
|
93
107
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
ErrorKind = Data.define(:name, :retryable)
|
|
5
|
+
|
|
6
|
+
##
|
|
7
|
+
# Closed classification of {Html2rss::Error} failures, so consumers branch on a stable
|
|
8
|
+
# symbol instead of exception class names. Read it through {Html2rss::Error#kind}.
|
|
9
|
+
class ErrorKind
|
|
10
|
+
# Frozen name → instance table. Only transient transport failures are retryable.
|
|
11
|
+
ALL = {
|
|
12
|
+
timeout: true,
|
|
13
|
+
blocked: false,
|
|
14
|
+
connection: true,
|
|
15
|
+
invalid_url: false,
|
|
16
|
+
budget: false,
|
|
17
|
+
other: false
|
|
18
|
+
}.to_h { |name, retryable| [name, new(name:, retryable:)] }.freeze
|
|
19
|
+
private_class_method :new
|
|
20
|
+
|
|
21
|
+
##
|
|
22
|
+
# @param name [Symbol, String] one of +ALL.keys+
|
|
23
|
+
# @return [ErrorKind]
|
|
24
|
+
# @raise [InvalidOption] when the name is not a known kind
|
|
25
|
+
def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown error kind: #{name.inspect}" }
|
|
26
|
+
|
|
27
|
+
##
|
|
28
|
+
# @return [Boolean] whether repeating the same request may succeed
|
|
29
|
+
def retryable? = retryable
|
|
30
|
+
|
|
31
|
+
##
|
|
32
|
+
# @return [Symbol]
|
|
33
|
+
def to_sym = name
|
|
34
|
+
|
|
35
|
+
##
|
|
36
|
+
# @return [String]
|
|
37
|
+
def to_s = name.to_s
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -5,11 +5,15 @@ module Html2rss
|
|
|
5
5
|
##
|
|
6
6
|
# Presentation rules for rendering an {Html2rss::Article} into feed wire formats.
|
|
7
7
|
#
|
|
8
|
-
# Owns description enrichment ({Html::Rendering::DescriptionBuilder})
|
|
9
|
-
#
|
|
8
|
+
# Owns description enrichment ({Html::Rendering::DescriptionBuilder}), RSS enclosure
|
|
9
|
+
# choice and the media-feed rule. {Article} keeps raw extracted fields only; media-file
|
|
10
|
+
# facts live on {Article::Enclosure}.
|
|
10
11
|
module ItemPresentation
|
|
11
12
|
class << self
|
|
12
13
|
##
|
|
14
|
+
# The item body: sanitized description text after at most one media element, the RSS enclosure
|
|
15
|
+
# ({.rss_enclosure_for}) when it is audio or video, else the item image ({Html::Rendering::DescriptionBuilder}).
|
|
16
|
+
#
|
|
13
17
|
# @param article [Html2rss::Article]
|
|
14
18
|
# @return [String, nil] sanitized description HTML/text for feed item bodies
|
|
15
19
|
def description_for(article)
|
|
@@ -17,24 +21,33 @@ module Html2rss
|
|
|
17
21
|
base: article.description,
|
|
18
22
|
title: article.title,
|
|
19
23
|
url: article.url,
|
|
20
|
-
|
|
24
|
+
enclosure: rss_enclosure_for(article),
|
|
21
25
|
image: article.image
|
|
22
26
|
).call
|
|
23
27
|
end
|
|
24
28
|
|
|
25
29
|
##
|
|
26
|
-
#
|
|
30
|
+
# The one RSS +enclosure+ of an item: its first audio or video file, else its first downloadable
|
|
31
|
+
# document or archive ({Article::Enclosure#document?}: PDF, ZIP). Images stay on the description /
|
|
32
|
+
# JSON Feed; stream playlists, Flash players and watch or player pages ({Article::Enclosure#player_page?})
|
|
33
|
+
# are never one.
|
|
27
34
|
#
|
|
28
35
|
# @param article [Html2rss::Article]
|
|
29
36
|
# @return [Html2rss::Article::Enclosure, nil]
|
|
30
37
|
def rss_enclosure_for(article)
|
|
31
|
-
|
|
38
|
+
enclosures = article.enclosures
|
|
39
|
+
enclosures.find(&:media?) || enclosures.find(&:document?)
|
|
32
40
|
end
|
|
33
41
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
42
|
+
##
|
|
43
|
+
# A media feed (podcast) is one where more than half of the items carry an audio or video
|
|
44
|
+
# RSS enclosure; only such a feed gets podcast (+itunes:+) tags.
|
|
45
|
+
#
|
|
46
|
+
# @param articles [Array<Html2rss::Article>]
|
|
47
|
+
# @return [Boolean]
|
|
48
|
+
def media_feed?(articles)
|
|
49
|
+
media_items = articles.count { |article| article.enclosures.any?(&:media?) }
|
|
50
|
+
media_items * 2 > articles.size
|
|
38
51
|
end
|
|
39
52
|
end
|
|
40
53
|
end
|
|
@@ -16,7 +16,7 @@ module Html2rss
|
|
|
16
16
|
# @param user_comment [String] required generator comment (from {Status})
|
|
17
17
|
# @param feed_url [String, nil] optional absolute self URL for the feed (JSON Feed feed_url)
|
|
18
18
|
def initialize(channel:, articles:, user_comment:, feed_url: nil)
|
|
19
|
-
raise
|
|
19
|
+
raise InvalidOption, 'user_comment must be a non-blank String' if blank_string?(user_comment)
|
|
20
20
|
|
|
21
21
|
@channel = channel
|
|
22
22
|
@articles = articles
|
|
@@ -42,7 +42,7 @@ module Html2rss
|
|
|
42
42
|
|
|
43
43
|
def normalize_feed_url(value)
|
|
44
44
|
return if value.nil?
|
|
45
|
-
raise
|
|
45
|
+
raise InvalidOption, 'feed_url must be a String or nil' unless value.is_a?(String)
|
|
46
46
|
return if value.strip.empty?
|
|
47
47
|
|
|
48
48
|
Url.from_absolute(value).to_s
|
|
@@ -43,9 +43,9 @@ module Html2rss
|
|
|
43
43
|
# @param type [String] MIME type (`text/css` or `text/xsl`)
|
|
44
44
|
# @param media [String] media query hint for the stylesheet
|
|
45
45
|
def initialize(href:, type:, media: 'all')
|
|
46
|
-
raise
|
|
47
|
-
raise
|
|
48
|
-
raise
|
|
46
|
+
raise InvalidOption, 'stylesheet.href must be a String' unless href.is_a?(String)
|
|
47
|
+
raise InvalidOption, 'stylesheet.type invalid' unless TYPES.include?(type)
|
|
48
|
+
raise InvalidOption, 'stylesheet.media must be a String' unless media.is_a?(String)
|
|
49
49
|
|
|
50
50
|
@href = href
|
|
51
51
|
@type = type
|