html2rss 0.32.0 → 0.32.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +7 -1
- data/lib/html2rss/auto_source/scraper/native_feed.rb +19 -4
- data/lib/html2rss/auto_source/scraper/schema.rb +6 -2
- data/lib/html2rss/auto_source/scraper/sitemap.rb +19 -4
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +6 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +20 -3
- data/lib/html2rss/auto_source.rb +6 -1
- data/lib/html2rss/error.rb +7 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +56 -15
- data/lib/html2rss/feed_resolution.rb +7 -4
- data/lib/html2rss/request_service/blocked_surface.rb +11 -0
- data/lib/html2rss/request_service/botasaurus_contract.rb +5 -0
- data/lib/html2rss/request_service/botasaurus_strategy.rb +10 -8
- data/lib/html2rss/request_service/httpx_strategy.rb +11 -1
- data/lib/html2rss/request_service/strategy.rb +9 -3
- data/lib/html2rss/request_service.rb +33 -8
- data/lib/html2rss/sst/normalizer.rb +20 -4
- data/lib/html2rss/syndication/discovery.rb +5 -1
- data/lib/html2rss/syndication/parser.rb +6 -2
- data/lib/html2rss/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e095abe52bc258d262e3a24909e0feb480b2eb1d537e57580fc271203b0c6dbc
|
|
4
|
+
data.tar.gz: 8fdabe7ed5afee08c46ab94c2b6f80c1f8c66df9ffec9439e40e80fa897bfa96
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fdcce48a2a4fb3a8c3ea02fd8f4bb799bd8ed7acb4e66e74da3cde693f6a51cc3aa72d363670b13f077d1521d97321e0676d14b59c6b61f1c685fdc4db64cb67
|
|
7
|
+
data.tar.gz: ce1c1cac4c38f4660e61cb7c9506991d8c0d847c8bdfbb583c678680854b7327a16b9aa637af641aaae42bed9f15b9330cc344ee922252da16863a517569fad4
|
|
@@ -116,6 +116,7 @@ module Html2rss
|
|
|
116
116
|
end
|
|
117
117
|
|
|
118
118
|
# @return [Hash{Symbol => Object}] oEmbed fields hash
|
|
119
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- structured warn keeps fields local
|
|
119
120
|
def fetch_oembed_data
|
|
120
121
|
return {} unless request_session && (link_node = oembed_link_node)
|
|
121
122
|
return {} unless (oembed_url = resolve_url(link_node['href']))
|
|
@@ -123,7 +124,12 @@ module Html2rss
|
|
|
123
124
|
response = request_session.follow_up(url: oembed_url, relation: :auto_source, origin_url: url)
|
|
124
125
|
parse_oembed_response(response)
|
|
125
126
|
rescue Html2rss::Error, RequestService::UnsupportedResponseContentType, JSON::ParserError => error
|
|
126
|
-
Log.warn(
|
|
127
|
+
Log.warn({
|
|
128
|
+
event_name: 'meta_oembed.fetch_failed',
|
|
129
|
+
host: url.host,
|
|
130
|
+
error_class: error.class.name,
|
|
131
|
+
message: "#{self.class}: failed to fetch oEmbed payload (#{error.class}: #{error.message})"
|
|
132
|
+
}.to_json)
|
|
127
133
|
{}
|
|
128
134
|
end
|
|
129
135
|
|
|
@@ -57,16 +57,25 @@ module Html2rss
|
|
|
57
57
|
##
|
|
58
58
|
# @yieldparam article [Hash{Symbol => Object}]
|
|
59
59
|
# @return [Enumerator, void]
|
|
60
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- structured empty/items logs
|
|
60
61
|
def each(&)
|
|
61
62
|
return enum_for(:each) unless block_given?
|
|
62
63
|
|
|
63
64
|
articles = fetch_articles
|
|
64
65
|
if articles.empty?
|
|
65
|
-
Log.info(
|
|
66
|
+
Log.info({
|
|
67
|
+
event_name: 'auto_source.native_feed.empty',
|
|
68
|
+
host: url.host,
|
|
69
|
+
message: "#{self.class}: host=#{url.host} item_count=0 fallback=true"
|
|
70
|
+
}.to_json)
|
|
66
71
|
return
|
|
67
72
|
end
|
|
68
73
|
|
|
69
|
-
Log.info(
|
|
74
|
+
Log.info({
|
|
75
|
+
event_name: 'auto_source.native_feed.items',
|
|
76
|
+
host: url.host,
|
|
77
|
+
message: "#{self.class}: host=#{url.host} item_count=#{articles.size} fallback=false"
|
|
78
|
+
}.to_json)
|
|
70
79
|
articles.each(&)
|
|
71
80
|
end
|
|
72
81
|
|
|
@@ -74,7 +83,8 @@ module Html2rss
|
|
|
74
83
|
|
|
75
84
|
attr_reader :parsed_body, :url, :request_session
|
|
76
85
|
|
|
77
|
-
|
|
86
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- discovery + parse + structured warn
|
|
87
|
+
def fetch_articles
|
|
78
88
|
return [] unless request_session
|
|
79
89
|
|
|
80
90
|
response = Syndication::Discovery.best_feed_response(
|
|
@@ -87,7 +97,12 @@ module Html2rss
|
|
|
87
97
|
|
|
88
98
|
Syndication::Parser.parse_response(response)
|
|
89
99
|
rescue Html2rss::Error, ArgumentError => error
|
|
90
|
-
Log.warn(
|
|
100
|
+
Log.warn({
|
|
101
|
+
event_name: 'auto_source.native_feed.failed',
|
|
102
|
+
host: url.host,
|
|
103
|
+
error_class: error.class.name,
|
|
104
|
+
message: "#{self.class}: host=#{url.host} failed (#{error.class}: #{error.message})"
|
|
105
|
+
}.to_json)
|
|
91
106
|
[]
|
|
92
107
|
end
|
|
93
108
|
end
|
|
@@ -12,7 +12,7 @@ module Html2rss
|
|
|
12
12
|
#
|
|
13
13
|
# @see https://schema.org/docs/full.html
|
|
14
14
|
# @see https://developers.google.com/search/docs/appearance/structured-data/article#microdata
|
|
15
|
-
class Schema
|
|
15
|
+
class Schema # rubocop:disable Metrics/ClassLength -- scraper + structured warn paths
|
|
16
16
|
include Enumerable
|
|
17
17
|
|
|
18
18
|
# Matches a leading schema.org URL prefix on @type values (http or https).
|
|
@@ -156,7 +156,11 @@ module Html2rss
|
|
|
156
156
|
def parse_script_tag(script_tag)
|
|
157
157
|
JSON.parse(script_tag.text, symbolize_names: true)
|
|
158
158
|
rescue JSON::ParserError => error
|
|
159
|
-
Log.warn(
|
|
159
|
+
Log.warn({
|
|
160
|
+
event_name: 'schema.parse_failed',
|
|
161
|
+
error_class: error.class.name,
|
|
162
|
+
message: "#{name}: failed to parse JSON (#{error.message})"
|
|
163
|
+
}.to_json)
|
|
160
164
|
[]
|
|
161
165
|
end
|
|
162
166
|
end
|
|
@@ -7,7 +7,7 @@ module Html2rss
|
|
|
7
7
|
module Scraper
|
|
8
8
|
##
|
|
9
9
|
# Scrapes articles from XML sitemaps explicitly linked in HTML or discovered via standard sitemap paths.
|
|
10
|
-
class Sitemap
|
|
10
|
+
class Sitemap # rubocop:disable Metrics/ClassLength -- scraper + structured warn paths
|
|
11
11
|
include Enumerable
|
|
12
12
|
|
|
13
13
|
# Selector for sitemap link tags in HTML head
|
|
@@ -78,6 +78,7 @@ module Html2rss
|
|
|
78
78
|
|
|
79
79
|
attr_reader :parsed_body, :body, :url, :request_session, :min_priority, :max_age_days
|
|
80
80
|
|
|
81
|
+
# rubocop:disable-next Metrics/MethodLength -- structured budget-exhausted warn
|
|
81
82
|
def fetch_and_parse_entries
|
|
82
83
|
entries = parse_direct_sitemap
|
|
83
84
|
return entries unless entries.empty?
|
|
@@ -87,7 +88,11 @@ module Html2rss
|
|
|
87
88
|
|
|
88
89
|
parse_sitemap_xml(xml_body)
|
|
89
90
|
rescue Html2rss::RequestService::RequestBudgetExceeded
|
|
90
|
-
Log.warn(
|
|
91
|
+
Log.warn({
|
|
92
|
+
event_name: 'sitemap.budget_exhausted',
|
|
93
|
+
host: url.host,
|
|
94
|
+
message: "#{self.class}: sitemap fetch stopped — request budget exhausted"
|
|
95
|
+
}.to_json)
|
|
91
96
|
[]
|
|
92
97
|
end
|
|
93
98
|
|
|
@@ -122,10 +127,15 @@ module Html2rss
|
|
|
122
127
|
|
|
123
128
|
Scraper::Sitemap::Parser.call(xml, min_priority:, max_age_days:).entries
|
|
124
129
|
rescue Html2rss::RequestService::RequestBudgetExceeded
|
|
125
|
-
Log.warn(
|
|
130
|
+
Log.warn({
|
|
131
|
+
event_name: 'sitemap.fanout_budget_exhausted',
|
|
132
|
+
host: url.host,
|
|
133
|
+
message: "#{self.class}: sitemap fan-out stopped — request budget exhausted"
|
|
134
|
+
}.to_json)
|
|
126
135
|
nil
|
|
127
136
|
end
|
|
128
137
|
|
|
138
|
+
# rubocop:disable-next Metrics/MethodLength -- structured fetch-failed warn
|
|
129
139
|
def fetch_xml(target_url)
|
|
130
140
|
return unless request_session && target_url
|
|
131
141
|
|
|
@@ -134,7 +144,12 @@ module Html2rss
|
|
|
134
144
|
rescue Html2rss::RequestService::RequestBudgetExceeded
|
|
135
145
|
raise
|
|
136
146
|
rescue Html2rss::Error => error
|
|
137
|
-
Log.warn(
|
|
147
|
+
Log.warn({
|
|
148
|
+
event_name: 'sitemap.fetch_failed',
|
|
149
|
+
host: url.host,
|
|
150
|
+
error_class: error.class.name,
|
|
151
|
+
message: "#{self.class}: failed to fetch sitemap (#{error.class}: #{error.message})"
|
|
152
|
+
}.to_json)
|
|
138
153
|
nil
|
|
139
154
|
end
|
|
140
155
|
|
|
@@ -59,7 +59,7 @@ module Html2rss
|
|
|
59
59
|
nil
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
-
def fetchable_page_scope?
|
|
62
|
+
def fetchable_page_scope? # rubocop:disable Metrics/MethodLength -- scope checks + structured warn
|
|
63
63
|
return true if page_scope.fetchable?
|
|
64
64
|
|
|
65
65
|
if page_scope.reason == :non_archive
|
|
@@ -70,7 +70,11 @@ module Html2rss
|
|
|
70
70
|
return false
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
-
logger.warn(
|
|
73
|
+
logger.warn({
|
|
74
|
+
event_name: 'wordpress_api.unsafe_archive_scope',
|
|
75
|
+
host: page_url.host,
|
|
76
|
+
message: "#{WordpressApi}: unable to derive safe WordPress archive scope for #{page_url}"
|
|
77
|
+
}.to_json)
|
|
74
78
|
false
|
|
75
79
|
end
|
|
76
80
|
|
|
@@ -75,19 +75,31 @@ module Html2rss
|
|
|
75
75
|
|
|
76
76
|
attr_reader :parsed_body, :url, :request_session, :page_scope
|
|
77
77
|
|
|
78
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- structured rescue logs
|
|
78
79
|
def fetch_posts
|
|
79
80
|
response = posts_response
|
|
80
81
|
return unless response
|
|
81
82
|
|
|
82
83
|
Array(response.parsed_body)
|
|
83
84
|
rescue RequestService::UnsupportedResponseContentType => error
|
|
84
|
-
Log.warn(
|
|
85
|
+
Log.warn({
|
|
86
|
+
event_name: 'wordpress_api.unsupported_content_type',
|
|
87
|
+
host: url.host,
|
|
88
|
+
error_class: error.class.name,
|
|
89
|
+
message: "#{self.class}: unsupported WordPress API posts content type (#{error.message})"
|
|
90
|
+
}.to_json)
|
|
85
91
|
nil
|
|
86
92
|
rescue JSON::ParserError => error
|
|
87
|
-
Log.warn(
|
|
93
|
+
Log.warn({
|
|
94
|
+
event_name: 'wordpress_api.parse_failed',
|
|
95
|
+
host: url.host,
|
|
96
|
+
error_class: error.class.name,
|
|
97
|
+
message: "#{self.class}: failed to parse WordPress API posts JSON (#{error.message})"
|
|
98
|
+
}.to_json)
|
|
88
99
|
nil
|
|
89
100
|
end
|
|
90
101
|
|
|
102
|
+
# rubocop:disable-next Metrics/MethodLength -- structured fetch-failed warn
|
|
91
103
|
def posts_response
|
|
92
104
|
return unless request_session
|
|
93
105
|
return unless (resolved_posts_url = posts_endpoint_url)
|
|
@@ -98,7 +110,12 @@ module Html2rss
|
|
|
98
110
|
origin_url: url
|
|
99
111
|
)
|
|
100
112
|
rescue Html2rss::Error => error
|
|
101
|
-
Log.warn(
|
|
113
|
+
Log.warn({
|
|
114
|
+
event_name: 'wordpress_api.fetch_failed',
|
|
115
|
+
host: url.host,
|
|
116
|
+
error_class: error.class.name,
|
|
117
|
+
message: "#{self.class}: failed to fetch WordPress API posts (#{error.class}: #{error.message})"
|
|
118
|
+
}.to_json)
|
|
102
119
|
nil
|
|
103
120
|
end
|
|
104
121
|
|
data/lib/html2rss/auto_source.rb
CHANGED
|
@@ -277,7 +277,12 @@ module Html2rss
|
|
|
277
277
|
def run_scraper(instance)
|
|
278
278
|
instance.map { |item| article_from_scraper_item(item, instance) }
|
|
279
279
|
rescue *OPERATIONAL_ERRORS => error
|
|
280
|
-
Log.warn(
|
|
280
|
+
Log.warn({
|
|
281
|
+
event_name: 'auto_source.quarantined',
|
|
282
|
+
host: url.host,
|
|
283
|
+
error_class: error.class.name,
|
|
284
|
+
message: "#{self.class}: #{instance.class} quarantined #{error.class}: #{error.message}"
|
|
285
|
+
}.to_json)
|
|
281
286
|
[]
|
|
282
287
|
end
|
|
283
288
|
|
data/lib/html2rss/error.rb
CHANGED
|
@@ -2,7 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
# The Html2rss::Error base class.
|
|
5
|
-
class Error < StandardError
|
|
5
|
+
class Error < StandardError
|
|
6
|
+
# @return [Array<Hash{Symbol => Object}>, nil] tier attempt diagnostics when set by AutoFallback
|
|
7
|
+
attr_accessor :attempts
|
|
8
|
+
|
|
9
|
+
# @return [Hash{String => Object}, nil] allowlisted upstream transport telemetry when attached
|
|
10
|
+
attr_accessor :transport_meta
|
|
11
|
+
end
|
|
6
12
|
|
|
7
13
|
# Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
|
|
8
14
|
class NoFeedItemsExtracted < Error
|
|
@@ -30,12 +30,13 @@ module Html2rss
|
|
|
30
30
|
##
|
|
31
31
|
# Mutable run state for one auto-fallback chain: attempts plus selected result.
|
|
32
32
|
class AttemptState
|
|
33
|
-
attr_reader :attempts, :result, :last_response, :entry_resolution
|
|
33
|
+
attr_reader :attempts, :result, :last_response, :last_error, :entry_resolution
|
|
34
34
|
|
|
35
35
|
def initialize
|
|
36
36
|
@attempts = []
|
|
37
37
|
@result = nil
|
|
38
38
|
@last_response = nil
|
|
39
|
+
@last_error = nil
|
|
39
40
|
@entry_resolution = nil
|
|
40
41
|
@resolution_tried = false
|
|
41
42
|
@aborted = false
|
|
@@ -70,7 +71,11 @@ module Html2rss
|
|
|
70
71
|
# @param error [Exception] caught error
|
|
71
72
|
# @return [void]
|
|
72
73
|
def record_error(strategy:, error:)
|
|
73
|
-
|
|
74
|
+
attempt = { strategy:, items_count: nil, error_class: error.class.name }
|
|
75
|
+
meta = error.is_a?(Html2rss::Error) ? error.transport_meta : nil
|
|
76
|
+
attempt[:transport_meta] = meta if meta&.any?
|
|
77
|
+
@attempts << attempt
|
|
78
|
+
@last_error = error
|
|
74
79
|
end
|
|
75
80
|
|
|
76
81
|
# @param strategy [Symbol] strategy that returned a response
|
|
@@ -78,6 +83,7 @@ module Html2rss
|
|
|
78
83
|
# @param transport_meta [Hash, nil] optional allowlisted upstream telemetry
|
|
79
84
|
# @return [void]
|
|
80
85
|
def record_items(strategy:, items_count:, transport_meta: nil)
|
|
86
|
+
@last_error = nil
|
|
81
87
|
attempt = { strategy:, items_count:, error_class: nil }
|
|
82
88
|
attempt[:transport_meta] = transport_meta if transport_meta && !transport_meta.empty?
|
|
83
89
|
@attempts << attempt
|
|
@@ -136,7 +142,7 @@ module Html2rss
|
|
|
136
142
|
state = run_attempts
|
|
137
143
|
return state.result if state.succeeded?
|
|
138
144
|
|
|
139
|
-
finalize_failure(
|
|
145
|
+
finalize_failure(state)
|
|
140
146
|
end
|
|
141
147
|
|
|
142
148
|
private
|
|
@@ -215,7 +221,7 @@ module Html2rss
|
|
|
215
221
|
pipeline.deduplicated_articles(config:, response:, request_session:)
|
|
216
222
|
end
|
|
217
223
|
|
|
218
|
-
# rubocop:disable-next Metrics/ParameterLists -- success kwargs
|
|
224
|
+
# rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- success kwargs + structured selected log
|
|
219
225
|
def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
|
|
220
226
|
attempt_count = state.attempts.size
|
|
221
227
|
state.succeed!(
|
|
@@ -225,14 +231,35 @@ module Html2rss
|
|
|
225
231
|
)
|
|
226
232
|
return unless attempt_count > 1
|
|
227
233
|
|
|
228
|
-
Log.info(
|
|
229
|
-
|
|
230
|
-
|
|
234
|
+
Log.info({
|
|
235
|
+
event_name: 'auto_fallback.selected',
|
|
236
|
+
host: response.url.host,
|
|
237
|
+
strategy: strategy.to_s,
|
|
238
|
+
attempt_count:,
|
|
239
|
+
message: "#{self.class}: auto selected strategy=#{strategy} after attempts=#{attempt_count} " \
|
|
240
|
+
"host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
|
|
241
|
+
"budget_remaining=#{budget_remaining_label}"
|
|
242
|
+
}.to_json)
|
|
231
243
|
end
|
|
232
244
|
|
|
233
|
-
def finalize_failure(
|
|
234
|
-
|
|
235
|
-
|
|
245
|
+
def finalize_failure(state)
|
|
246
|
+
if raise_last_error?(state)
|
|
247
|
+
error = state.last_error
|
|
248
|
+
error.attempts = state.attempts if error.is_a?(Html2rss::Error)
|
|
249
|
+
raise error
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
raise NoFeedItemsExtracted.new(
|
|
253
|
+
attempts: state.attempts,
|
|
254
|
+
surface_category: surface_category_for(state.last_response)
|
|
255
|
+
)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def raise_last_error?(state)
|
|
259
|
+
error = state.last_error
|
|
260
|
+
return false if error.nil? || error.is_a?(RequestService::BotasaurusConfigurationError)
|
|
261
|
+
|
|
262
|
+
true
|
|
236
263
|
end
|
|
237
264
|
|
|
238
265
|
def surface_category_for(response)
|
|
@@ -247,8 +274,15 @@ module Html2rss
|
|
|
247
274
|
detail = "host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
|
|
248
275
|
"budget_remaining=#{budget_remaining_label}"
|
|
249
276
|
if error.is_a?(RequestService::RequestTimedOut)
|
|
250
|
-
Log.info(
|
|
251
|
-
|
|
277
|
+
Log.info({
|
|
278
|
+
event_name: 'auto_fallback.timeout',
|
|
279
|
+
host:,
|
|
280
|
+
strategy: strategy.to_s,
|
|
281
|
+
next_strategy: next_strategy.to_s,
|
|
282
|
+
error_class: error.class.name,
|
|
283
|
+
message: "#{self.class}: auto fallback #{strategy} -> #{next_strategy} " \
|
|
284
|
+
"after timeout=#{error.class} #{detail}"
|
|
285
|
+
}.to_json)
|
|
252
286
|
else
|
|
253
287
|
Log.warn({
|
|
254
288
|
event_name: 'auto_fallback.error',
|
|
@@ -264,9 +298,16 @@ module Html2rss
|
|
|
264
298
|
end
|
|
265
299
|
|
|
266
300
|
def log_info_fallback_zero_items(strategy:, next_strategy:, response:)
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
301
|
+
host = response.url.host
|
|
302
|
+
Log.info({
|
|
303
|
+
event_name: 'auto_fallback.zero_items',
|
|
304
|
+
host:,
|
|
305
|
+
strategy: strategy.to_s,
|
|
306
|
+
next_strategy: next_strategy.to_s,
|
|
307
|
+
message: "#{self.class}: auto fallback #{strategy} -> #{next_strategy} after zero extracted items " \
|
|
308
|
+
"host=#{host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
|
|
309
|
+
"budget_remaining=#{budget_remaining_label}"
|
|
310
|
+
}.to_json)
|
|
270
311
|
end
|
|
271
312
|
|
|
272
313
|
def budget_remaining_label
|
|
@@ -285,10 +285,13 @@ module Html2rss
|
|
|
285
285
|
|
|
286
286
|
def result(applied:, reason:, scrape_url:, probe_count:, winner_score:)
|
|
287
287
|
scrape_host = Html2rss::Url.from_absolute(scrape_url).host
|
|
288
|
-
Log.info(
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
288
|
+
Log.info({
|
|
289
|
+
event_name: 'feed_resolution.result',
|
|
290
|
+
host: entry_url.host,
|
|
291
|
+
message: "FeedResolution: entry_host=#{entry_url.host} scrape_host=#{scrape_host} " \
|
|
292
|
+
"applied=#{applied} probe_count=#{probe_count} winner_score=#{winner_score.inspect} " \
|
|
293
|
+
"reason=#{reason}"
|
|
294
|
+
}.to_json)
|
|
292
295
|
Result.new(entry_url: entry_url.to_s, scrape_url:, applied:, reason:, probe_count:, winner_score:)
|
|
293
296
|
end
|
|
294
297
|
end
|
|
@@ -47,6 +47,17 @@ module Html2rss
|
|
|
47
47
|
message: 'Blocked surface detected: Vercel Security Checkpoint. ' \
|
|
48
48
|
'This site is a JS-rendered SPA behind Vercel edge protection. ' \
|
|
49
49
|
'Configure BOTASAURUS_SCRAPER_URL or target a direct listing URL.'
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
key: :http_403_forbidden,
|
|
53
|
+
min_matches: 2,
|
|
54
|
+
patterns: [
|
|
55
|
+
%r{<title>\s*403\s+forbidden\s*</title>}i,
|
|
56
|
+
%r{<h1>\s*403\s+forbidden\s*</h1>}i
|
|
57
|
+
],
|
|
58
|
+
message: 'Blocked surface detected: HTTP 403 Forbidden access block page. ' \
|
|
59
|
+
'Target a direct listing URL, configure BOTASAURUS_SCRAPER_URL, ' \
|
|
60
|
+
'or run from an environment that is not IP-blocked.'
|
|
50
61
|
}
|
|
51
62
|
].freeze
|
|
52
63
|
|
|
@@ -198,6 +198,11 @@ module Html2rss
|
|
|
198
198
|
"Botasaurus scrape failed (#{details.join(', ')})."
|
|
199
199
|
end
|
|
200
200
|
|
|
201
|
+
# @return [Hash{String => Object}] allowlisted diagnostics plus error_category (frozen)
|
|
202
|
+
def transport_meta
|
|
203
|
+
diagnostics.merge('error_category' => error_category).freeze
|
|
204
|
+
end
|
|
205
|
+
|
|
201
206
|
private
|
|
202
207
|
|
|
203
208
|
attr_reader :payload, :transport_status
|
|
@@ -49,14 +49,16 @@ module Html2rss
|
|
|
49
49
|
end
|
|
50
50
|
|
|
51
51
|
def raise_from_error!(error)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
52
|
+
err = if error.challenge_block?
|
|
53
|
+
BlockedSurfaceDetected.new("Blocked surface detected: #{error.challenge_message}")
|
|
54
|
+
elsif error.timeout?
|
|
55
|
+
log_timeout!(reason: 'botasaurus_upstream')
|
|
56
|
+
RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
|
|
57
|
+
else
|
|
58
|
+
BotasaurusServiceError.new(error.failure_message)
|
|
59
|
+
end
|
|
60
|
+
err.transport_meta = error.transport_meta
|
|
61
|
+
raise err
|
|
60
62
|
end
|
|
61
63
|
|
|
62
64
|
def response_url(final_url)
|
|
@@ -60,8 +60,18 @@ module Html2rss
|
|
|
60
60
|
|
|
61
61
|
private
|
|
62
62
|
|
|
63
|
+
# Caps one HTTPX execute (including internal MAX_RETRIES) so a hanging
|
|
64
|
+
# :default tier cannot consume the full shared auto-fallback budget.
|
|
65
|
+
#
|
|
66
|
+
# @return [Float] monotonic deadline
|
|
63
67
|
def request_deadline
|
|
64
|
-
|
|
68
|
+
budget_timeout = ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
|
|
69
|
+
monotonic_now + [budget_timeout, attempt_timeout_ceiling].min
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# @return [Float] connect + read ceiling for a single attempt
|
|
73
|
+
def attempt_timeout_ceiling
|
|
74
|
+
(ctx.policy.connect_timeout_seconds + ctx.policy.read_timeout_seconds).to_f
|
|
65
75
|
end
|
|
66
76
|
|
|
67
77
|
def reset_redirect_tracking!
|
|
@@ -111,18 +111,24 @@ module Html2rss
|
|
|
111
111
|
|
|
112
112
|
# @param reason [String] timeout classification (budget_exhausted / transport)
|
|
113
113
|
# @return [void]
|
|
114
|
-
# rubocop:disable-next Metrics/AbcSize -- structured timeout fields stay in one log line
|
|
114
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- structured timeout fields stay in one log line
|
|
115
115
|
def log_timeout!(reason:)
|
|
116
116
|
remaining = ctx.budget.remaining_timeout_seconds
|
|
117
117
|
remaining_label = remaining.nil? ? 'untracked' : format('%.3f', remaining)
|
|
118
|
+
host = ctx.url.host
|
|
118
119
|
detail = [
|
|
119
120
|
"strategy=#{self.class.name}",
|
|
120
|
-
"host=#{
|
|
121
|
+
"host=#{host}",
|
|
121
122
|
"elapsed=#{format('%.3f', ctx.budget.elapsed_seconds)}s",
|
|
122
123
|
"budget_remaining=#{remaining_label}",
|
|
123
124
|
"reason=#{reason}"
|
|
124
125
|
]
|
|
125
|
-
Log.info(
|
|
126
|
+
Log.info({
|
|
127
|
+
event_name: 'request_service.timeout',
|
|
128
|
+
host:,
|
|
129
|
+
strategy: self.class.name,
|
|
130
|
+
message: "#{self.class}: request timeout #{detail.join(' ')}"
|
|
131
|
+
}.to_json)
|
|
126
132
|
end
|
|
127
133
|
|
|
128
134
|
# @param error [StandardError]
|
|
@@ -98,24 +98,49 @@ module Html2rss
|
|
|
98
98
|
# @return [Response] the response from the executed strategy.
|
|
99
99
|
# @raise [UnknownStrategy] if the strategy is not registered.
|
|
100
100
|
def execute(ctx, strategy: default_strategy_name)
|
|
101
|
-
strategy_sym = strategy.
|
|
101
|
+
strategy_sym = resolve_strategy_name(strategy, relation: ctx.relation, host: ctx.url.host)
|
|
102
102
|
strategy_class = STRATEGIES.fetch(strategy_sym) do
|
|
103
103
|
raise UnknownStrategy,
|
|
104
104
|
"The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
|
|
105
105
|
end
|
|
106
106
|
|
|
107
|
-
warn_migration_strategy(strategy_sym) if MIGRATION_REPLACEMENTS.key?(strategy_sym)
|
|
108
107
|
strategy_class.new(ctx).execute
|
|
109
108
|
end
|
|
110
109
|
|
|
111
110
|
private
|
|
112
111
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
112
|
+
##
|
|
113
|
+
# Resolves deprecated migration aliases to their replacement strategy name.
|
|
114
|
+
# @param strategy [Symbol, String] requested strategy name
|
|
115
|
+
# @param relation [Symbol] request relation (+ :initial + or follow-up)
|
|
116
|
+
# @param host [String, nil] request host for structured deprecation telemetry
|
|
117
|
+
# @return [Symbol] canonical strategy name after migration remapping
|
|
118
|
+
def resolve_strategy_name(strategy, relation:, host:)
|
|
119
|
+
strategy_sym = strategy.to_sym
|
|
120
|
+
replacement = MIGRATION_REPLACEMENTS[strategy_sym]
|
|
121
|
+
return strategy_sym unless replacement
|
|
122
|
+
|
|
123
|
+
warn_migration_strategy(strategy_sym, replacement, relation:, host:)
|
|
124
|
+
replacement
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
##
|
|
128
|
+
# Emits a structured +strategy_deprecated+ warning once per session (+ :initial + only).
|
|
129
|
+
# @param strategy [Symbol] deprecated strategy alias
|
|
130
|
+
# @param replacement [Symbol] preferred strategy name
|
|
131
|
+
# @param relation [Symbol] request relation
|
|
132
|
+
# @param host [String, nil] request host
|
|
133
|
+
# @return [void]
|
|
134
|
+
def warn_migration_strategy(strategy, replacement, relation:, host:)
|
|
135
|
+
return unless relation == :initial
|
|
136
|
+
|
|
137
|
+
Log.warn({
|
|
138
|
+
event_name: 'strategy_deprecated',
|
|
139
|
+
strategy: strategy.to_s,
|
|
140
|
+
replacement: replacement.to_s,
|
|
141
|
+
host:,
|
|
142
|
+
message: "#{name}: strategy=#{strategy.inspect} is deprecated; using #{replacement.inspect} instead."
|
|
143
|
+
}.to_json)
|
|
119
144
|
end
|
|
120
145
|
end
|
|
121
146
|
end
|
|
@@ -119,17 +119,29 @@ module Html2rss
|
|
|
119
119
|
|
|
120
120
|
body = parsed.at_css('body')
|
|
121
121
|
if body
|
|
122
|
-
Html2rss::Log.warn(
|
|
122
|
+
Html2rss::Log.warn({
|
|
123
|
+
event_name: 'sst.normalizer.root_fallback',
|
|
124
|
+
reason: 'body',
|
|
125
|
+
message: 'sst.normalizer root fallback: body'
|
|
126
|
+
}.to_json)
|
|
123
127
|
return body
|
|
124
128
|
end
|
|
125
129
|
|
|
126
130
|
first = parsed.element_children.find { |child| element_root?(child) }
|
|
127
131
|
if first
|
|
128
|
-
Html2rss::Log.warn(
|
|
132
|
+
Html2rss::Log.warn({
|
|
133
|
+
event_name: 'sst.normalizer.root_fallback',
|
|
134
|
+
reason: 'first_element_child',
|
|
135
|
+
message: 'sst.normalizer root fallback: first element child'
|
|
136
|
+
}.to_json)
|
|
129
137
|
return first
|
|
130
138
|
end
|
|
131
139
|
|
|
132
|
-
Html2rss::Log.warn(
|
|
140
|
+
Html2rss::Log.warn({
|
|
141
|
+
event_name: 'sst.normalizer.root_fallback',
|
|
142
|
+
reason: 'self',
|
|
143
|
+
message: 'sst.normalizer root fallback: self'
|
|
144
|
+
}.to_json)
|
|
133
145
|
parsed
|
|
134
146
|
end
|
|
135
147
|
|
|
@@ -146,7 +158,11 @@ module Html2rss
|
|
|
146
158
|
|
|
147
159
|
if @node_count >= MAX_NODES && !@degraded
|
|
148
160
|
@degraded = true
|
|
149
|
-
Html2rss::Log.warn(
|
|
161
|
+
Html2rss::Log.warn({
|
|
162
|
+
event_name: 'sst.normalizer.max_nodes',
|
|
163
|
+
max_nodes: MAX_NODES,
|
|
164
|
+
message: "sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only"
|
|
165
|
+
}.to_json)
|
|
150
166
|
end
|
|
151
167
|
|
|
152
168
|
return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
|
|
@@ -60,7 +60,11 @@ module Html2rss
|
|
|
60
60
|
).tap do |response|
|
|
61
61
|
next unless response
|
|
62
62
|
|
|
63
|
-
Log.info(
|
|
63
|
+
Log.info({
|
|
64
|
+
event_name: 'syndication.discovery.selected',
|
|
65
|
+
host: page.host,
|
|
66
|
+
message: "Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}"
|
|
67
|
+
}.to_json)
|
|
64
68
|
end
|
|
65
69
|
end
|
|
66
70
|
|
|
@@ -6,7 +6,7 @@ module Html2rss
|
|
|
6
6
|
module Syndication
|
|
7
7
|
##
|
|
8
8
|
# Parses RSS 2.0 and Atom documents into AutoSource article hashes.
|
|
9
|
-
module Parser
|
|
9
|
+
module Parser # rubocop:disable Metrics/ModuleLength -- feed formats + structured parse warn
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
12
|
##
|
|
@@ -29,7 +29,11 @@ module Html2rss
|
|
|
29
29
|
|
|
30
30
|
items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
|
|
31
31
|
rescue RSS::Error => error
|
|
32
|
-
Log.warn(
|
|
32
|
+
Log.warn({
|
|
33
|
+
event_name: 'syndication.parse_failed',
|
|
34
|
+
error_class: error.class.name,
|
|
35
|
+
message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
|
|
36
|
+
}.to_json)
|
|
33
37
|
[]
|
|
34
38
|
end
|
|
35
39
|
|
data/lib/html2rss/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html2rss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.32.
|
|
4
|
+
version: 0.32.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gil Desmarais
|
|
@@ -478,7 +478,7 @@ licenses:
|
|
|
478
478
|
- MIT
|
|
479
479
|
metadata:
|
|
480
480
|
allowed_push_host: https://rubygems.org
|
|
481
|
-
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.
|
|
481
|
+
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.2
|
|
482
482
|
rubygems_mfa_required: 'true'
|
|
483
483
|
rdoc_options: []
|
|
484
484
|
require_paths:
|