html2rss 0.33.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/lib/html2rss/article/deduplicator.rb +5 -2
- data/lib/html2rss/article/enclosure.rb +185 -21
- data/lib/html2rss/article/title.rb +203 -0
- data/lib/html2rss/article.rb +79 -13
- data/lib/html2rss/auto_source/README.md +16 -8
- data/lib/html2rss/auto_source/cleanup.rb +126 -50
- data/lib/html2rss/auto_source/corroboration.rb +167 -0
- data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
- data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
- data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
- data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
- data/lib/html2rss/auto_source/page_links.rb +164 -0
- data/lib/html2rss/auto_source/scraper/html.rb +11 -39
- data/lib/html2rss/auto_source/scraper/json_document.rb +1 -3
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +119 -42
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +5 -16
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +2 -10
- data/lib/html2rss/auto_source/scraper/json_state.rb +2 -3
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +6 -14
- data/lib/html2rss/auto_source/scraper/microdata.rb +6 -4
- data/lib/html2rss/auto_source/scraper/microformats2.rb +1 -3
- data/lib/html2rss/auto_source/scraper/native_feed.rb +27 -17
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +68 -27
- data/lib/html2rss/auto_source/scraper/schema.rb +5 -8
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +1 -5
- data/lib/html2rss/auto_source/scraper/sitemap.rb +4 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -3
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -3
- data/lib/html2rss/auto_source/scraper.rb +10 -2
- data/lib/html2rss/auto_source/segment.rb +19 -5
- data/lib/html2rss/auto_source/segmenter/cluster.rb +13 -11
- data/lib/html2rss/auto_source/segmenter/list.rb +13 -5
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
- data/lib/html2rss/auto_source/segmenter/semantic.rb +17 -6
- data/lib/html2rss/auto_source/segmenter.rb +4 -8
- data/lib/html2rss/auto_source.rb +81 -26
- data/lib/html2rss/batch.rb +2 -0
- data/lib/html2rss/capture/README.md +5 -2
- data/lib/html2rss/capture.rb +44 -32
- data/lib/html2rss/channel.rb +30 -12
- data/lib/html2rss/cli.rb +22 -11
- data/lib/html2rss/config/auto_source_contract.rb +52 -45
- data/lib/html2rss/config/issue_mapper.rb +4 -1
- data/lib/html2rss/config/request_controls.rb +6 -3
- data/lib/html2rss/config/schema.rb +15 -5
- data/lib/html2rss/config/selectors_validator.rb +2 -2
- data/lib/html2rss/config/validation_issue.rb +4 -1
- data/lib/html2rss/config/validator.rb +11 -0
- data/lib/html2rss/config.rb +53 -27
- data/lib/html2rss/contract_error.rb +9 -0
- data/lib/html2rss/defaults.rb +21 -23
- data/lib/html2rss/doctor/botasaurus.rb +7 -3
- data/lib/html2rss/error.rb +10 -0
- data/lib/html2rss/error_kind.rb +39 -0
- data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
- data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
- data/lib/html2rss/feed_builder/json_feed.rb +2 -2
- data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
- data/lib/html2rss/feed_builder/rss.rb +49 -6
- data/lib/html2rss/feed_builder.rb +2 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +13 -31
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
- data/lib/html2rss/feed_pipeline.rb +40 -27
- data/lib/html2rss/feed_resolution/README.md +1 -1
- data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
- data/lib/html2rss/feed_resolution/policy.rb +1 -1
- data/lib/html2rss/feed_resolution/probe.rb +3 -1
- data/lib/html2rss/feed_resolution.rb +11 -12
- data/lib/html2rss/feed_result.rb +9 -5
- data/lib/html2rss/hash_util.rb +3 -3
- data/lib/html2rss/html/article_extractor/category_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -31
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +6 -9
- data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
- data/lib/html2rss/html/article_extractor.rb +19 -47
- data/lib/html2rss/html/article_rules/category.rb +3 -3
- data/lib/html2rss/html/article_rules/description.rb +6 -15
- data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
- data/lib/html2rss/html/article_rules/image.rb +30 -16
- data/lib/html2rss/html/feed_link.rb +7 -9
- data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
- data/lib/html2rss/html/navigator.rb +39 -9
- data/lib/html2rss/html/probe.rb +4 -0
- data/lib/html2rss/html/rendering/description_builder.rb +46 -32
- data/lib/html2rss/html/rendering.rb +2 -9
- data/lib/html2rss/html/sst_article_extractor.rb +55 -145
- data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
- data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
- data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
- data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
- data/lib/html2rss/html/title_pieces.rb +169 -0
- data/lib/html2rss/html/visibility.rb +56 -0
- data/lib/html2rss/html.rb +2 -0
- data/lib/html2rss/link_destination/destination_facts.rb +17 -1
- data/lib/html2rss/link_destination/noise_policy.rb +11 -10
- data/lib/html2rss/link_destination/path_classifier.rb +100 -9
- data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
- data/lib/html2rss/link_destination/sponsorship.rb +184 -0
- data/lib/html2rss/link_destination/text_classifier.rb +104 -8
- data/lib/html2rss/mcp/README.md +13 -1
- data/lib/html2rss/mcp/config_argument.rb +5 -2
- data/lib/html2rss/mcp/contract.rb +13 -4
- data/lib/html2rss/mcp/outcome.rb +14 -12
- data/lib/html2rss/mcp/runtime.rb +39 -2
- data/lib/html2rss/mcp/server.rb +13 -4
- data/lib/html2rss/page_recon/README.md +1 -1
- data/lib/html2rss/page_recon/diagnostics.rb +2 -0
- data/lib/html2rss/recon.rb +10 -12
- data/lib/html2rss/request_service/blocked_surface.rb +2 -1
- data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
- data/lib/html2rss/request_service/botasaurus_strategy.rb +42 -10
- data/lib/html2rss/request_service/context.rb +15 -14
- data/lib/html2rss/request_service/cookie_jar.rb +89 -0
- data/lib/html2rss/request_service/cookie_store.rb +79 -0
- data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
- data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +2 -2
- data/lib/html2rss/request_service/policy.rb +48 -11
- data/lib/html2rss/request_service/replay_breaker.rb +56 -0
- data/lib/html2rss/request_service/response.rb +30 -13
- data/lib/html2rss/request_service/response_cache.rb +109 -0
- data/lib/html2rss/request_service/response_caching.rb +95 -0
- data/lib/html2rss/request_service/strategy.rb +5 -11
- data/lib/html2rss/request_service.rb +71 -12
- data/lib/html2rss/request_session/pager/base.rb +0 -7
- data/lib/html2rss/request_session/pager.rb +1 -1
- data/lib/html2rss/request_session.rb +6 -2
- data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
- data/lib/html2rss/scoring/container_assessor.rb +12 -13
- data/lib/html2rss/scoring/engine.rb +6 -6
- data/lib/html2rss/scoring/observation.rb +2 -2
- data/lib/html2rss/selector_candidates.rb +19 -8
- data/lib/html2rss/selectors/README.md +1 -1
- data/lib/html2rss/selectors/option_spec.rb +4 -1
- data/lib/html2rss/selectors/post_processors/base.rb +0 -8
- data/lib/html2rss/selectors/post_processors/gsub.rb +7 -7
- data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +3 -3
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
- data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
- data/lib/html2rss/selectors.rb +53 -12
- data/lib/html2rss/sst/attrs.rb +2 -2
- data/lib/html2rss/sst/document.rb +3 -3
- data/lib/html2rss/sst/index.rb +30 -11
- data/lib/html2rss/sst/node.rb +33 -26
- data/lib/html2rss/sst/normalizer.rb +31 -19
- data/lib/html2rss/sst/tags.rb +113 -2
- data/lib/html2rss/sst/text.rb +69 -29
- data/lib/html2rss/sst.rb +2 -0
- data/lib/html2rss/status.rb +6 -3
- data/lib/html2rss/surface_category.rb +1 -1
- data/lib/html2rss/syndication/README.md +12 -6
- data/lib/html2rss/syndication/alternate_links.rb +1 -1
- data/lib/html2rss/syndication/channel_facts.rb +87 -0
- data/lib/html2rss/syndication/discovery.rb +82 -28
- data/lib/html2rss/syndication/feed.rb +20 -0
- data/lib/html2rss/syndication/item_media.rb +103 -0
- data/lib/html2rss/syndication/parser.rb +75 -51
- data/lib/html2rss/test/enhance_audit.rb +2 -2
- data/lib/html2rss/test/item_sample.rb +43 -0
- data/lib/html2rss/test.rb +42 -14
- data/lib/html2rss/url.rb +115 -14
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss/word_count.rb +53 -0
- data/lib/html2rss.rb +17 -8
- data/schema/html2rss-config.schema.json +56 -41
- metadata +31 -4
- data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
- data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
|
@@ -16,7 +16,7 @@ module Html2rss
|
|
|
16
16
|
# @param user_comment [String] required generator comment (from {Status})
|
|
17
17
|
# @param feed_url [String, nil] optional absolute self URL for the feed (JSON Feed feed_url)
|
|
18
18
|
def initialize(channel:, articles:, user_comment:, feed_url: nil)
|
|
19
|
-
raise
|
|
19
|
+
raise InvalidOption, 'user_comment must be a non-blank String' if blank_string?(user_comment)
|
|
20
20
|
|
|
21
21
|
@channel = channel
|
|
22
22
|
@articles = articles
|
|
@@ -42,7 +42,7 @@ module Html2rss
|
|
|
42
42
|
|
|
43
43
|
def normalize_feed_url(value)
|
|
44
44
|
return if value.nil?
|
|
45
|
-
raise
|
|
45
|
+
raise InvalidOption, 'feed_url must be a String or nil' unless value.is_a?(String)
|
|
46
46
|
return if value.strip.empty?
|
|
47
47
|
|
|
48
48
|
Url.from_absolute(value).to_s
|
|
@@ -43,9 +43,9 @@ module Html2rss
|
|
|
43
43
|
# @param type [String] MIME type (`text/css` or `text/xsl`)
|
|
44
44
|
# @param media [String] media query hint for the stylesheet
|
|
45
45
|
def initialize(href:, type:, media: 'all')
|
|
46
|
-
raise
|
|
47
|
-
raise
|
|
48
|
-
raise
|
|
46
|
+
raise InvalidOption, 'stylesheet.href must be a String' unless href.is_a?(String)
|
|
47
|
+
raise InvalidOption, 'stylesheet.type invalid' unless TYPES.include?(type)
|
|
48
|
+
raise InvalidOption, 'stylesheet.media must be a String' unless media.is_a?(String)
|
|
49
49
|
|
|
50
50
|
@href = href
|
|
51
51
|
@type = type
|
|
@@ -6,7 +6,20 @@ module Html2rss
|
|
|
6
6
|
module FeedBuilder
|
|
7
7
|
##
|
|
8
8
|
# Builds an RSS Feed by providing channel, articles and stylesheets.
|
|
9
|
-
|
|
9
|
+
#
|
|
10
|
+
# A media feed ({ItemPresentation.media_feed?}) also gets the podcast tags clients play from:
|
|
11
|
+
# channel +itunes:image+, +itunes:author+, +itunes:category+ (nesting its subcategories) and
|
|
12
|
+
# +itunes:explicit+ (+true+ or +false+), item +itunes:duration+ (+HH:MM:SS+) and +itunes:image+, each
|
|
13
|
+
# only when known; categories and the explicit flag only when the followed native feed states them.
|
|
14
|
+
#
|
|
15
|
+
# +managingEditor+ carries the channel author only when it is an RSS 2.0 email address
|
|
16
|
+
# (+editor@example.com+ or +editor@example.com (Name)+); a plain name is no managing editor.
|
|
17
|
+
class Rss # rubocop:disable Metrics/ClassLength -- channel, item and podcast projections stay co-located
|
|
18
|
+
# First duration in seconds that +itunes:duration+ cannot spell as +HH:MM:SS+ (100 hours).
|
|
19
|
+
MAX_CLOCK_DURATION = 100 * 3600
|
|
20
|
+
# An RSS 2.0 person: an email address, optionally followed by the name in parentheses.
|
|
21
|
+
EMAIL_PERSON = /\A[^\s@()]+@[^\s@()]+(?: \(.+\))?\z/
|
|
22
|
+
|
|
10
23
|
class << self
|
|
11
24
|
# @param article [Html2rss::Article] source article
|
|
12
25
|
# @param item_maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
|
|
@@ -60,12 +73,13 @@ module Html2rss
|
|
|
60
73
|
# @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
|
|
61
74
|
# @param generator [String] required preformatted generator comment (from {Status})
|
|
62
75
|
def initialize(channel:, articles:, generator:, stylesheets: [])
|
|
63
|
-
raise
|
|
76
|
+
raise InvalidOption, 'generator must be a non-blank String' if blank_string?(generator)
|
|
64
77
|
|
|
65
78
|
@channel = channel
|
|
66
79
|
@articles = articles
|
|
67
80
|
@stylesheets = stylesheets
|
|
68
81
|
@generator = generator
|
|
82
|
+
@media_feed = ItemPresentation.media_feed?(articles)
|
|
69
83
|
end
|
|
70
84
|
|
|
71
85
|
# @return [RSS::Rss] RSS 2.0 document instance
|
|
@@ -80,7 +94,7 @@ module Html2rss
|
|
|
80
94
|
|
|
81
95
|
private
|
|
82
96
|
|
|
83
|
-
attr_reader :channel, :articles, :generator
|
|
97
|
+
attr_reader :channel, :articles, :generator, :media_feed
|
|
84
98
|
|
|
85
99
|
def blank_string?(value)
|
|
86
100
|
!value.is_a?(String) || value.strip.empty?
|
|
@@ -98,13 +112,25 @@ module Html2rss
|
|
|
98
112
|
channel_maker.description = channel.description
|
|
99
113
|
channel_maker.ttl = channel.ttl
|
|
100
114
|
|
|
101
|
-
channel_maker.managingEditor = channel.author if channel.author
|
|
102
|
-
channel_maker.author = channel.author if channel.author
|
|
115
|
+
channel_maker.managingEditor = channel.author if EMAIL_PERSON.match?(channel.author)
|
|
103
116
|
channel_maker.link = channel.url.to_s
|
|
104
117
|
channel_maker.generator = generator
|
|
105
118
|
channel_maker.updated = channel.last_build_date
|
|
106
119
|
|
|
107
120
|
make_image(maker.image) if channel.image
|
|
121
|
+
make_podcast_channel(channel_maker) if media_feed
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def make_podcast_channel(channel_maker)
|
|
125
|
+
channel_maker.itunes_image = channel.image
|
|
126
|
+
channel_maker.itunes_author = channel.author
|
|
127
|
+
channel_maker.itunes_explicit = channel.explicit&.to_s
|
|
128
|
+
channel.categories.each { add_itunes_category(channel_maker, *_1) }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def add_itunes_category(channel_maker, name, *subcategories)
|
|
132
|
+
category = channel_maker.new_itunes_category(name)
|
|
133
|
+
subcategories.each { category.new_category.text = _1 }
|
|
108
134
|
end
|
|
109
135
|
|
|
110
136
|
def make_image(image_maker)
|
|
@@ -114,9 +140,26 @@ module Html2rss
|
|
|
114
140
|
|
|
115
141
|
def make_items(maker)
|
|
116
142
|
articles.each do |article|
|
|
117
|
-
maker.items.new_item
|
|
143
|
+
maker.items.new_item do |item_maker|
|
|
144
|
+
self.class.add_item(article, item_maker)
|
|
145
|
+
make_podcast_item(article, item_maker) if media_feed
|
|
146
|
+
end
|
|
118
147
|
end
|
|
119
148
|
end
|
|
149
|
+
|
|
150
|
+
def make_podcast_item(article, item_maker)
|
|
151
|
+
duration = ItemPresentation.rss_enclosure_for(article)&.duration
|
|
152
|
+
item_maker.itunes_duration = format_duration(duration) if duration
|
|
153
|
+
item_maker.itunes_image = article.image
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# +HH:MM:SS+; plain seconds from 100 hours on, which +HH+ cannot spell.
|
|
157
|
+
def format_duration(seconds)
|
|
158
|
+
return seconds.to_s if seconds >= MAX_CLOCK_DURATION
|
|
159
|
+
|
|
160
|
+
format('%<hours>02d:%<minutes>02d:%<seconds>02d',
|
|
161
|
+
hours: seconds / 3600, minutes: seconds % 3600 / 60, seconds: seconds % 60)
|
|
162
|
+
end
|
|
120
163
|
end
|
|
121
164
|
end
|
|
122
165
|
end
|
|
@@ -8,7 +8,8 @@ module Html2rss
|
|
|
8
8
|
# FeedResult; +feed_url+ is a render-time JSON Feed-only option.
|
|
9
9
|
#
|
|
10
10
|
# Channel field projection:
|
|
11
|
-
# - Rss: language, title, description, ttl, link, updated
|
|
11
|
+
# - Rss: language, title, description, ttl, link, updated, managingEditor (author in email form only)
|
|
12
|
+
# (+ itunes:image, itunes:author, itunes:category, itunes:explicit on a media feed)
|
|
12
13
|
# - JsonFeed: title, home_page_url, description, language, icon, authors (+ feed_url)
|
|
13
14
|
module FeedBuilder
|
|
14
15
|
end
|
|
@@ -11,7 +11,7 @@ Use `:auto` when you want `default` (HTTPX) first and a browser-backed hop only
|
|
|
11
11
|
1. **Default (HTTPX)** — plain HTTP (faster, cheaper).
|
|
12
12
|
2. **Botasaurus** — attempted when default raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
|
|
13
13
|
|
|
14
|
-
Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
|
|
14
|
+
Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom/RDF entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
|
|
15
15
|
|
|
16
16
|
There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without default first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
|
|
17
17
|
|
|
@@ -108,26 +108,15 @@ module Html2rss
|
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
# @param response [RequestService::Response] successful response
|
|
111
|
-
# @param
|
|
112
|
-
# @param dedup_dropped [Integer] articles removed by deduplication
|
|
111
|
+
# @param extraction [Html2rss::FeedPipeline::Extraction] its articles, drops and feed channel facts
|
|
113
112
|
# @param selected_strategy [Symbol] concrete strategy that produced items
|
|
114
113
|
# @param attempt_count [Integer] number of attempts recorded for this chain
|
|
115
|
-
# @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
|
|
116
114
|
# @param scrape_target [Html2rss::ScrapeTarget]
|
|
117
115
|
# @return [void]
|
|
118
|
-
|
|
119
|
-
def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
|
|
120
|
-
scrape_target:, admission_drops: {})
|
|
116
|
+
def succeed!(response:, extraction:, selected_strategy:, attempt_count:, scrape_target:)
|
|
121
117
|
@result = PipelineOutcome.new(
|
|
122
|
-
response:,
|
|
123
|
-
|
|
124
|
-
dedup_dropped:,
|
|
125
|
-
selected_strategy:,
|
|
126
|
-
attempt_count:,
|
|
127
|
-
strategy_attempts: attempts,
|
|
128
|
-
admission_drops:,
|
|
129
|
-
scrape_target:,
|
|
130
|
-
entry_resolution:
|
|
118
|
+
response:, selected_strategy:, attempt_count:, strategy_attempts: attempts,
|
|
119
|
+
scrape_target:, entry_resolution:, **extraction.to_h
|
|
131
120
|
)
|
|
132
121
|
end
|
|
133
122
|
|
|
@@ -215,6 +204,8 @@ module Html2rss
|
|
|
215
204
|
rescue *NON_FALLBACK_ERRORS => error
|
|
216
205
|
state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
|
|
217
206
|
state.abort!
|
|
207
|
+
rescue Html2rss::ContractError
|
|
208
|
+
raise
|
|
218
209
|
rescue Html2rss::Error => error
|
|
219
210
|
state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
|
|
220
211
|
log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy && request_session
|
|
@@ -222,11 +213,9 @@ module Html2rss
|
|
|
222
213
|
|
|
223
214
|
# rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Metrics/ParameterLists -- extract + resolution delegate
|
|
224
215
|
def process_response(response:, strategy:, next_strategy:, request_session:, state:, elapsed_ms: nil)
|
|
225
|
-
|
|
226
|
-
config:, response:, request_session:
|
|
227
|
-
)
|
|
216
|
+
extraction = pipeline.extract(config:, response:, request_session:)
|
|
228
217
|
state.remember_response(response)
|
|
229
|
-
items_count = articles.size
|
|
218
|
+
items_count = extraction.articles.size
|
|
230
219
|
state.record_items(strategy:, items_count:, transport_meta: response.transport_meta, elapsed_ms:)
|
|
231
220
|
Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
|
|
232
221
|
"host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
|
|
@@ -234,16 +223,13 @@ module Html2rss
|
|
|
234
223
|
|
|
235
224
|
if (outcome = FeedResolution.try_apply!(
|
|
236
225
|
pipeline:, config:, response:, session: request_session, strategy:, resources:,
|
|
237
|
-
articles
|
|
226
|
+
articles: extraction.articles, scrape_target:, state:, budget:
|
|
238
227
|
))
|
|
239
228
|
@scrape_target = outcome.scrape_target
|
|
240
229
|
return
|
|
241
230
|
end
|
|
242
231
|
|
|
243
|
-
if items_count.positive?
|
|
244
|
-
return record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:,
|
|
245
|
-
state:)
|
|
246
|
-
end
|
|
232
|
+
return record_success(response:, strategy:, extraction:, state:) if items_count.positive?
|
|
247
233
|
|
|
248
234
|
if DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
|
|
249
235
|
state.abort!
|
|
@@ -254,14 +240,10 @@ module Html2rss
|
|
|
254
240
|
log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
|
|
255
241
|
end
|
|
256
242
|
|
|
257
|
-
# rubocop:disable-next Metrics/
|
|
258
|
-
def record_success(response:, strategy:,
|
|
243
|
+
# rubocop:disable-next Metrics/MethodLength -- success + structured selected log
|
|
244
|
+
def record_success(response:, strategy:, extraction:, state:)
|
|
259
245
|
attempt_count = state.attempts.size
|
|
260
|
-
state.succeed!(
|
|
261
|
-
response:, articles:, dedup_dropped:, selected_strategy: strategy,
|
|
262
|
-
attempt_count:, admission_drops:,
|
|
263
|
-
scrape_target:
|
|
264
|
-
)
|
|
246
|
+
state.succeed!(response:, extraction:, selected_strategy: strategy, attempt_count:, scrape_target:)
|
|
265
247
|
return unless attempt_count > 1
|
|
266
248
|
|
|
267
249
|
Log.info({
|
|
@@ -33,13 +33,13 @@ module Html2rss
|
|
|
33
33
|
##
|
|
34
34
|
# @param name [Symbol, String] config or configure strategy value
|
|
35
35
|
# @return [Auto, Concrete] resolved feed-level plan
|
|
36
|
-
# @raise [
|
|
36
|
+
# @raise [InvalidOption] when +name+ is neither +:auto+ nor a registered strategy
|
|
37
37
|
def resolve(name)
|
|
38
38
|
normalized = name.to_sym
|
|
39
39
|
return Auto.new if normalized == AUTO_NAME
|
|
40
40
|
|
|
41
41
|
unless RequestService.strategy_registered?(normalized)
|
|
42
|
-
raise
|
|
42
|
+
raise InvalidOption, "unknown strategy plan: #{name.inspect}"
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
Concrete.new(strategy: normalized)
|
|
@@ -64,7 +64,7 @@ module Html2rss
|
|
|
64
64
|
|
|
65
65
|
resolve(name)
|
|
66
66
|
true
|
|
67
|
-
rescue
|
|
67
|
+
rescue InvalidOption
|
|
68
68
|
false
|
|
69
69
|
end
|
|
70
70
|
|
|
@@ -10,11 +10,19 @@ module Html2rss
|
|
|
10
10
|
# selected_strategy: set on :auto success; nil otherwise.
|
|
11
11
|
# attempt_count: auto attempts attempted; 0 outside :auto.
|
|
12
12
|
# strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
|
|
13
|
+
# feed_channel: channel facts of the native feed followed ({Syndication::ChannelFacts}); nil without one.
|
|
13
14
|
PipelineOutcome = Data.define(
|
|
14
15
|
:response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
|
|
15
|
-
:admission_drops, :scrape_target, :entry_resolution
|
|
16
|
+
:admission_drops, :scrape_target, :entry_resolution, :feed_channel
|
|
16
17
|
)
|
|
17
18
|
|
|
19
|
+
# What one response yields after extraction and dedup ({#extract}); its fields are the
|
|
20
|
+
# {PipelineOutcome} fields of the same names.
|
|
21
|
+
# articles: unique articles. dedup_dropped: articles removed by deduplication.
|
|
22
|
+
# admission_drops: Cleanup drop tallies. feed_channel: the followed native feed's channel facts
|
|
23
|
+
# ({Syndication::ChannelFacts}); nil without one.
|
|
24
|
+
Extraction = Data.define(:articles, :dedup_dropped, :admission_drops, :feed_channel)
|
|
25
|
+
|
|
18
26
|
##
|
|
19
27
|
# @param raw_config [Hash{Symbol => Object}] user-provided feed config
|
|
20
28
|
def initialize(raw_config)
|
|
@@ -71,11 +79,11 @@ module Html2rss
|
|
|
71
79
|
# @param config [Html2rss::Config]
|
|
72
80
|
# @param response [Html2rss::RequestService::Response]
|
|
73
81
|
# @param request_session [Html2rss::RequestSession]
|
|
74
|
-
# @return [
|
|
75
|
-
def
|
|
76
|
-
collected, admission_drops = collect_articles(config:, response:, request_session:)
|
|
82
|
+
# @return [Extraction] unique articles with their dedup and admission drops and feed channel facts
|
|
83
|
+
def extract(config:, response:, request_session:)
|
|
84
|
+
collected, admission_drops, feed_channel = collect_articles(config:, response:, request_session:)
|
|
77
85
|
unique = Article::Deduplicator.new(collected).call
|
|
78
|
-
|
|
86
|
+
Extraction.new(articles: unique, dedup_dropped: collected.size - unique.size, admission_drops:, feed_channel:)
|
|
79
87
|
end
|
|
80
88
|
|
|
81
89
|
private
|
|
@@ -95,13 +103,12 @@ module Html2rss
|
|
|
95
103
|
def run_pipeline_for_strategy(config, strategy:, resources:)
|
|
96
104
|
request_session = request_session_for(config, strategy:, resources:)
|
|
97
105
|
response = request_session.fetch_initial_response
|
|
98
|
-
|
|
99
|
-
raise_empty_auto_source!(strategy:, response:) if config.auto_source && articles.empty?
|
|
106
|
+
extraction = extract(config:, response:, request_session:)
|
|
107
|
+
raise_empty_auto_source!(strategy:, response:) if config.auto_source && extraction.articles.empty?
|
|
100
108
|
|
|
101
109
|
PipelineOutcome.new(
|
|
102
|
-
response:,
|
|
103
|
-
|
|
104
|
-
scrape_target: ScrapeTarget.from_config(config), entry_resolution: nil
|
|
110
|
+
response:, selected_strategy: nil, attempt_count: 0, strategy_attempts: [],
|
|
111
|
+
scrape_target: ScrapeTarget.from_config(config), entry_resolution: nil, **extraction.to_h
|
|
105
112
|
)
|
|
106
113
|
end
|
|
107
114
|
|
|
@@ -127,49 +134,55 @@ module Html2rss
|
|
|
127
134
|
end
|
|
128
135
|
|
|
129
136
|
def collect_articles(config:, response:, request_session:)
|
|
130
|
-
selector = selector_articles(config:, response:, request_session:)
|
|
131
|
-
auto,
|
|
132
|
-
[selector + auto,
|
|
137
|
+
selector, selector_drops = selector_articles(config:, response:, request_session:)
|
|
138
|
+
auto, auto_drops, feed_channel = auto_source_articles(config:, response:, request_session:)
|
|
139
|
+
[selector + auto, selector_drops.merge(auto_drops) { |_reason, left, right| left + right }, feed_channel]
|
|
133
140
|
end
|
|
134
141
|
|
|
135
|
-
#
|
|
142
|
+
# @return [Array(Array<Html2rss::Article>, Hash{String => Integer})] articles and unlinked-item drop tallies
|
|
136
143
|
def selector_articles(config:, response:, request_session:)
|
|
137
|
-
return [] unless (selectors = config.selectors)
|
|
138
|
-
return [] if response.feed_response?
|
|
144
|
+
return [[], {}] unless (selectors = config.selectors)
|
|
145
|
+
return [[], {}] if response.feed_response?
|
|
139
146
|
|
|
140
147
|
page_responses = request_session.page_responses(
|
|
141
148
|
response,
|
|
142
149
|
pagination_config: selectors.dig(:items, :pagination)
|
|
143
150
|
)
|
|
151
|
+
engines = page_responses.lazy.map { |page| Selectors.new(page, selectors:, time_zone: config.time_zone) }
|
|
152
|
+
collect_selector_pages(engines)
|
|
153
|
+
end
|
|
144
154
|
|
|
155
|
+
# Stops at the first empty page after articles were found; drop tallies sum over the pages read.
|
|
156
|
+
def collect_selector_pages(engines)
|
|
145
157
|
articles = []
|
|
146
|
-
|
|
147
|
-
|
|
158
|
+
drops = Hash.new(0)
|
|
159
|
+
engines.each do |engine|
|
|
160
|
+
page_articles = engine.articles
|
|
161
|
+
engine.admission_drops.each { |reason, count| drops[reason] += count }
|
|
148
162
|
break if page_articles.empty? && articles.any?
|
|
149
163
|
|
|
150
164
|
articles.concat(page_articles)
|
|
151
165
|
end
|
|
152
|
-
articles
|
|
166
|
+
[articles, drops]
|
|
153
167
|
end
|
|
154
168
|
|
|
155
|
-
# @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
|
|
169
|
+
# @return [Array(Array<Html2rss::Article>, Hash{String => Integer}, Html2rss::Syndication::ChannelFacts)]
|
|
156
170
|
def auto_source_articles(config:, response:, request_session:)
|
|
157
|
-
return [[], {}] unless (auto_source = config.auto_source)
|
|
171
|
+
return [[], {}, nil] unless (auto_source = config.auto_source)
|
|
158
172
|
return syndication_articles(response) if response.feed_response?
|
|
159
173
|
|
|
160
174
|
source = AutoSource.new(response, auto_source, request_session:)
|
|
161
|
-
[source.articles, source.admission_drops]
|
|
175
|
+
[source.articles, source.admission_drops, source.feed_channel]
|
|
162
176
|
end
|
|
163
177
|
|
|
164
178
|
def syndication_articles(response)
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
[articles, {}]
|
|
179
|
+
feed = Syndication::Parser.parse_response(response)
|
|
180
|
+
articles = feed.articles.map { |hash| Article.new(**hash, scraper: AutoSource::Scraper::NativeFeed) }
|
|
181
|
+
[articles, {}, feed.channel]
|
|
169
182
|
end
|
|
170
183
|
|
|
171
184
|
def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
|
|
172
|
-
channel = Channel.from_response(outcome.response, overrides: config.channel)
|
|
185
|
+
channel = Channel.from_response(outcome.response, overrides: config.channel, feed: outcome.feed_channel)
|
|
173
186
|
status = Status.build(
|
|
174
187
|
articles: outcome.articles,
|
|
175
188
|
dedup_dropped: outcome.dedup_dropped,
|
|
@@ -24,7 +24,7 @@ tournament on item count (feed-as-means) — intentional.
|
|
|
24
24
|
|
|
25
25
|
### CandidateGenerator mix
|
|
26
26
|
|
|
27
|
-
- **Feed slot:** first `Syndication::Discovery.candidate_urls`
|
|
27
|
+
- **Feed slot:** first same-domain `Syndication::Discovery.candidate_urls` entry. Alternates precede path guesses, except audio-hinted ones on a page without podcast signals. An alternate on another domain is skipped.
|
|
28
28
|
- **Listing fill** (cap = `max - 1` when `max > 1`; `max == 1` → feed only): nav (`taxonomy_path +3`, `content_path +1`) → segment first-wins (`:list` → `:cluster` → `:semantic`, ≥2 same-origin primary links) → `LISTING_PATHS`.
|
|
29
29
|
- Same-page / same-registrable-domain filters; feed never pads into listing slots.
|
|
30
30
|
|
|
@@ -46,13 +46,12 @@ module Html2rss
|
|
|
46
46
|
|
|
47
47
|
def listing_cap = max > 1 ? max - 1 : 0
|
|
48
48
|
|
|
49
|
+
# First eligible Discovery candidate: an alternate on another domain (Discovery ranks
|
|
50
|
+
# audio-hinted ones first on a podcast page) must not displace a same-domain feed or path guess.
|
|
49
51
|
def feed_slot
|
|
50
52
|
return unless parsed_html
|
|
51
53
|
|
|
52
|
-
|
|
53
|
-
return unless eligible_url?(url)
|
|
54
|
-
|
|
55
|
-
url
|
|
54
|
+
Syndication::Discovery.candidate_urls(page_url: entry_url, parsed_body: parsed_html).find { eligible_url?(_1) }
|
|
56
55
|
end
|
|
57
56
|
|
|
58
57
|
def listing_urls
|
|
@@ -15,7 +15,7 @@ module Html2rss
|
|
|
15
15
|
# @param surface_category [Html2rss::SurfaceCategory, Symbol]
|
|
16
16
|
# @return [Boolean]
|
|
17
17
|
def resolve?(config:, articles:, surface_category:)
|
|
18
|
-
raise
|
|
18
|
+
raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
|
|
19
19
|
return false unless eligible_config?(config)
|
|
20
20
|
|
|
21
21
|
category = SurfaceCategory[surface_category]
|
|
@@ -29,6 +29,8 @@ module Html2rss
|
|
|
29
29
|
origin_url:
|
|
30
30
|
)
|
|
31
31
|
score_response(response)
|
|
32
|
+
rescue Html2rss::ContractError
|
|
33
|
+
raise
|
|
32
34
|
rescue Html2rss::Error
|
|
33
35
|
nil
|
|
34
36
|
end
|
|
@@ -39,7 +41,7 @@ module Html2rss
|
|
|
39
41
|
|
|
40
42
|
def score_response(response) # rubocop:disable Metrics/MethodLength -- feed vs HTML branches
|
|
41
43
|
if response.feed_response?
|
|
42
|
-
count = Syndication::Parser.parse_response(response).size
|
|
44
|
+
count = Syndication::Parser.parse_response(response).articles.size
|
|
43
45
|
return Scored.new(
|
|
44
46
|
url: response.url,
|
|
45
47
|
score: Scorer.score_feed(articles_count: count),
|
|
@@ -8,6 +8,9 @@ module Html2rss
|
|
|
8
8
|
#
|
|
9
9
|
# {include:file:lib/html2rss/feed_resolution/README.md}
|
|
10
10
|
module FeedResolution
|
|
11
|
+
# Raised when the tournament or its policy is handed articles that are not an Array.
|
|
12
|
+
class InvalidError < Html2rss::ContractError; end
|
|
13
|
+
|
|
11
14
|
# Default entry-resolution options merged into AutoSource::DEFAULT_CONFIG.
|
|
12
15
|
DEFAULT_CONFIG = {
|
|
13
16
|
enabled: true,
|
|
@@ -120,7 +123,7 @@ module Html2rss
|
|
|
120
123
|
rescue RequestService::RequestBudgetExceeded
|
|
121
124
|
log_budget_exhausted
|
|
122
125
|
nil
|
|
123
|
-
rescue *NON_FALLBACK_ERRORS
|
|
126
|
+
rescue *NON_FALLBACK_ERRORS, Html2rss::ContractError
|
|
124
127
|
raise
|
|
125
128
|
rescue Html2rss::Error => error
|
|
126
129
|
log_retry_failed(error)
|
|
@@ -185,19 +188,15 @@ module Html2rss
|
|
|
185
188
|
config, strategy:, resources:, scrape_url: effective.effective_url
|
|
186
189
|
)
|
|
187
190
|
retry_response = retry_session.fetch_initial_response
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
)
|
|
191
|
+
extraction = pipeline.extract(config:, response: retry_response, request_session: retry_session)
|
|
192
|
+
items_count = extraction.articles.size
|
|
191
193
|
state.remember_response(retry_response)
|
|
192
|
-
state.record_items(
|
|
193
|
-
|
|
194
|
-
)
|
|
195
|
-
return unless retry_articles.size.positive?
|
|
194
|
+
state.record_items(strategy:, items_count:, transport_meta: retry_response.transport_meta)
|
|
195
|
+
return unless items_count.positive?
|
|
196
196
|
|
|
197
197
|
state.succeed!(
|
|
198
|
-
response: retry_response,
|
|
199
|
-
|
|
200
|
-
scrape_target: effective
|
|
198
|
+
response: retry_response, extraction:, selected_strategy: strategy,
|
|
199
|
+
attempt_count: state.attempts.size, scrape_target: effective
|
|
201
200
|
)
|
|
202
201
|
true
|
|
203
202
|
end
|
|
@@ -214,7 +213,7 @@ module Html2rss
|
|
|
214
213
|
# @param surface_category [Symbol, nil]
|
|
215
214
|
# rubocop:disable-next Metrics/ParameterLists -- tournament context stays co-located
|
|
216
215
|
def initialize(entry_url:, response:, session:, config:, articles:, surface_category: nil)
|
|
217
|
-
raise
|
|
216
|
+
raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
|
|
218
217
|
|
|
219
218
|
@entry_url = Url.from_absolute(entry_url)
|
|
220
219
|
@response = response
|
data/lib/html2rss/feed_result.rb
CHANGED
|
@@ -15,16 +15,20 @@ module Html2rss
|
|
|
15
15
|
# are re-frozen via private {#marshal_load}. For trusted cache reads only,
|
|
16
16
|
# +Marshal.load(payload, freeze: true)+ is also safe.
|
|
17
17
|
class FeedResult
|
|
18
|
+
# Raised when the scrape handle is built from invalid parts.
|
|
19
|
+
class InvalidError < Html2rss::ContractError; end
|
|
20
|
+
|
|
18
21
|
##
|
|
19
22
|
# @param channel [Html2rss::Channel] channel metadata (internal; not exposed on the public API)
|
|
20
23
|
# @param articles [Array<Html2rss::Article>] extracted articles (deduplicated; not exposed)
|
|
21
24
|
# @param status [Html2rss::Status] pipeline telemetry
|
|
22
25
|
# @param stylesheets [Array<Hash>] optional RSS stylesheet configs
|
|
26
|
+
# @raise [InvalidError] when a part has the wrong type
|
|
23
27
|
def initialize(channel:, articles:, status:, stylesheets: [])
|
|
24
|
-
raise
|
|
25
|
-
raise
|
|
26
|
-
raise
|
|
27
|
-
raise
|
|
28
|
+
raise InvalidError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
|
|
29
|
+
raise InvalidError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
|
|
30
|
+
raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
|
|
31
|
+
raise InvalidError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
|
|
28
32
|
|
|
29
33
|
@channel = channel
|
|
30
34
|
@articles = articles.dup.freeze
|
|
@@ -81,7 +85,7 @@ module Html2rss
|
|
|
81
85
|
end
|
|
82
86
|
|
|
83
87
|
def freeze_stylesheets(stylesheets)
|
|
84
|
-
raise
|
|
88
|
+
raise InvalidError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
|
|
85
89
|
|
|
86
90
|
stylesheets.map do |sheet|
|
|
87
91
|
sheet.to_h.transform_keys(&:to_sym).transform_values do |value|
|
data/lib/html2rss/hash_util.rb
CHANGED
|
@@ -81,7 +81,7 @@ module Html2rss
|
|
|
81
81
|
|
|
82
82
|
unless value.each_key.all?(Symbol)
|
|
83
83
|
invalid_key = value.keys.find { _1.class != Symbol }
|
|
84
|
-
raise
|
|
84
|
+
raise InvalidOption, "#{context} must use symbol keys (found #{invalid_key.inspect})"
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
|
|
@@ -98,7 +98,7 @@ module Html2rss
|
|
|
98
98
|
|
|
99
99
|
unless value.each_key.all?(String)
|
|
100
100
|
invalid_key = value.keys.find { _1.class != String }
|
|
101
|
-
raise
|
|
101
|
+
raise InvalidOption, "#{context} must use string keys (found #{invalid_key.inspect})"
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
|
|
@@ -111,7 +111,7 @@ module Html2rss
|
|
|
111
111
|
in Symbol then key
|
|
112
112
|
in String then key.to_sym
|
|
113
113
|
else
|
|
114
|
-
raise
|
|
114
|
+
raise InvalidOption, "#{context} must use string or symbol keys (found #{key.inspect})"
|
|
115
115
|
end
|
|
116
116
|
end
|
|
117
117
|
end
|
|
@@ -42,7 +42,7 @@ module Html2rss
|
|
|
42
42
|
Set.new.tap do |categories|
|
|
43
43
|
extract_element_data_categories!(categories, article_tag, title:)
|
|
44
44
|
article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
|
|
45
|
-
if ArticleRules::Category.class_match?(element['class'])
|
|
45
|
+
if ArticleRules::Category.class_match?(element['class'].to_s.split)
|
|
46
46
|
extract_text_categories!(categories, element, title:)
|
|
47
47
|
end
|
|
48
48
|
extract_element_data_categories!(categories, element, title:)
|