html2rss 0.32.4 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/html2rss/article/enclosure.rb +2 -6
- data/lib/html2rss/auto_source/cleanup.rb +1 -3
- data/lib/html2rss/auto_source/scraper/html.rb +2 -8
- data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
- data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
- data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
- data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
- data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
- data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
- data/lib/html2rss/auto_source/scraper.rb +0 -11
- data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
- data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
- data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
- data/lib/html2rss/auto_source.rb +1 -1
- data/lib/html2rss/batch.rb +22 -32
- data/lib/html2rss/capture/README.md +1 -1
- data/lib/html2rss/capture.rb +7 -27
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +33 -27
- data/lib/html2rss/config/request_controls.rb +10 -2
- data/lib/html2rss/config/schema.rb +8 -10
- data/lib/html2rss/config/selectors_validator.rb +1 -0
- data/lib/html2rss/config/validator.rb +3 -3
- data/lib/html2rss/config.rb +1 -1
- data/lib/html2rss/defaults.rb +5 -16
- data/lib/html2rss/doctor/botasaurus.rb +11 -19
- data/lib/html2rss/error.rb +7 -3
- data/lib/html2rss/feed_builder/rss.rb +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +3 -3
- data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +3 -3
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +5 -9
- data/lib/html2rss/feed_result.rb +4 -3
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
- data/lib/html2rss/html/probe.rb +51 -51
- data/lib/html2rss/link_destination/path_classifier.rb +11 -33
- data/lib/html2rss/mcp/config_argument.rb +9 -2
- data/lib/html2rss/mcp/contract.rb +7 -5
- data/lib/html2rss/mcp/outcome.rb +4 -5
- data/lib/html2rss/mcp/runtime.rb +17 -24
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/diagnostics.rb +7 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +10 -12
- data/lib/html2rss/request_service/blocked_surface.rb +3 -7
- data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +1 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +0 -1
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +8 -26
- data/lib/html2rss/request_service/response.rb +3 -6
- data/lib/html2rss/request_session/pager/base.rb +3 -3
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/scoring/engine.rb +4 -32
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +14 -18
- data/lib/html2rss/sst/node.rb +9 -3
- data/lib/html2rss/sst/normalizer.rb +25 -24
- data/lib/html2rss/sst/text.rb +53 -57
- data/lib/html2rss/status.rb +23 -30
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/discovery.rb +117 -187
- data/lib/html2rss/syndication/parser.rb +92 -115
- data/lib/html2rss/test/enhance_audit.rb +13 -15
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +29 -48
- data/lib/html2rss/url.rb +53 -53
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +2 -1
- metadata +4 -2
|
@@ -6,136 +6,113 @@ module Html2rss
|
|
|
6
6
|
module Syndication
|
|
7
7
|
##
|
|
8
8
|
# Parses RSS 2.0 and Atom documents into AutoSource article hashes.
|
|
9
|
-
module Parser
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
9
|
+
module Parser
|
|
10
|
+
class << self
|
|
11
|
+
##
|
|
12
|
+
# @param response [Html2rss::RequestService::Response]
|
|
13
|
+
# @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
|
|
14
|
+
# @raise [ArgumentError] when the response is not a syndication feed
|
|
15
|
+
def parse_response(response)
|
|
16
|
+
raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
|
|
17
|
+
|
|
18
|
+
parse(response.body, base_url: response.url)
|
|
19
|
+
end
|
|
18
20
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
+
##
|
|
22
|
+
# @param body [String] raw RSS/Atom document
|
|
23
|
+
# @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
|
|
24
|
+
# @return [Array<Hash{Symbol => Object}>]
|
|
25
|
+
def parse(body, base_url: nil)
|
|
26
|
+
feed = RSS::Parser.parse(body.to_s, false)
|
|
27
|
+
|
|
28
|
+
items_from(feed).filter_map { |item| article_hash(item, base_url:) }
|
|
29
|
+
rescue RSS::Error => error
|
|
30
|
+
Log.warn({
|
|
31
|
+
event_name: 'syndication.parse_failed',
|
|
32
|
+
error_class: error.class.name,
|
|
33
|
+
message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
|
|
34
|
+
}.to_json)
|
|
35
|
+
[]
|
|
36
|
+
end
|
|
21
37
|
|
|
22
|
-
|
|
23
|
-
# @param body [String] raw RSS/Atom document
|
|
24
|
-
# @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
|
|
25
|
-
# @return [Array<Hash{Symbol => Object}>]
|
|
26
|
-
def parse(body, base_url: nil)
|
|
27
|
-
feed = RSS::Parser.parse(body.to_s, false)
|
|
28
|
-
return [] unless feed
|
|
29
|
-
|
|
30
|
-
items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
|
|
31
|
-
rescue RSS::Error => error
|
|
32
|
-
Log.warn({
|
|
33
|
-
event_name: 'syndication.parse_failed',
|
|
34
|
-
error_class: error.class.name,
|
|
35
|
-
message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
|
|
36
|
-
}.to_json)
|
|
37
|
-
[]
|
|
38
|
-
end
|
|
38
|
+
private
|
|
39
39
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
40
|
+
def items_from(feed)
|
|
41
|
+
case feed
|
|
42
|
+
when RSS::Rss then feed.items
|
|
43
|
+
when RSS::Atom::Feed then feed.entries
|
|
44
|
+
else []
|
|
45
|
+
end
|
|
45
46
|
end
|
|
46
|
-
end
|
|
47
|
-
module_function :items_from
|
|
48
|
-
private_class_method :items_from
|
|
49
47
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
48
|
+
def article_hash(item, base_url:)
|
|
49
|
+
case item
|
|
50
|
+
when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
|
|
51
|
+
when RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
|
|
52
|
+
end
|
|
54
53
|
end
|
|
55
|
-
end
|
|
56
|
-
module_function :article_hash
|
|
57
|
-
private_class_method :article_hash
|
|
58
|
-
|
|
59
|
-
def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
|
|
60
|
-
url = absolute_url(item.link, base_url:)
|
|
61
|
-
return unless url
|
|
62
|
-
|
|
63
|
-
{
|
|
64
|
-
id: present(item.guid&.content) || url.to_s,
|
|
65
|
-
title: present(item.title),
|
|
66
|
-
description: present(item.description),
|
|
67
|
-
url:,
|
|
68
|
-
published_at: item.date || item.pubDate,
|
|
69
|
-
author: present(item.author),
|
|
70
|
-
categories: Array(item.categories).filter_map { |cat| present(cat.content) }
|
|
71
|
-
}.compact
|
|
72
|
-
end
|
|
73
|
-
module_function :rss_item_hash
|
|
74
|
-
private_class_method :rss_item_hash
|
|
75
|
-
|
|
76
|
-
def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
|
|
77
|
-
url = atom_link(entry, base_url:)
|
|
78
|
-
return unless url
|
|
79
|
-
|
|
80
|
-
{
|
|
81
|
-
id: present(entry.id&.content) || url.to_s,
|
|
82
|
-
title: present(atom_text(entry.title)),
|
|
83
|
-
description: present(atom_text(entry.content) || atom_text(entry.summary)),
|
|
84
|
-
url:,
|
|
85
|
-
published_at: entry.published&.content || entry.updated&.content,
|
|
86
|
-
author: present(atom_author(entry)),
|
|
87
|
-
categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
|
|
88
|
-
}.compact
|
|
89
|
-
end
|
|
90
|
-
module_function :atom_entry_hash
|
|
91
|
-
private_class_method :atom_entry_hash
|
|
92
54
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
55
|
+
def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
|
|
56
|
+
url = absolute_url(item.link, base_url:)
|
|
57
|
+
return unless url
|
|
58
|
+
|
|
59
|
+
{
|
|
60
|
+
id: present(item.guid&.content) || url.to_s,
|
|
61
|
+
title: present(item.title),
|
|
62
|
+
description: present(item.description),
|
|
63
|
+
url:,
|
|
64
|
+
published_at: item.pubDate,
|
|
65
|
+
author: present(item.author),
|
|
66
|
+
categories: item.categories.filter_map { |cat| present(cat.content) }
|
|
67
|
+
}.compact
|
|
68
|
+
end
|
|
100
69
|
|
|
101
|
-
|
|
102
|
-
|
|
70
|
+
def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
|
|
71
|
+
url = atom_link(entry, base_url:)
|
|
72
|
+
return unless url
|
|
73
|
+
|
|
74
|
+
{
|
|
75
|
+
id: present(entry.id&.content) || url.to_s,
|
|
76
|
+
title: present(entry.title&.content),
|
|
77
|
+
description: present(entry.content&.content || entry.summary&.content),
|
|
78
|
+
url:,
|
|
79
|
+
published_at: entry.published&.content || entry.updated&.content,
|
|
80
|
+
author: atom_author(entry),
|
|
81
|
+
categories: entry.categories.filter_map { |cat| present(cat.term) }
|
|
82
|
+
}.compact
|
|
83
|
+
end
|
|
103
84
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
85
|
+
def atom_link(entry, base_url:)
|
|
86
|
+
links = entry.links
|
|
87
|
+
link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
|
|
88
|
+
absolute_url(link&.href, base_url:)
|
|
89
|
+
end
|
|
108
90
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
91
|
+
def atom_author(entry)
|
|
92
|
+
author = entry.author
|
|
93
|
+
return unless author
|
|
112
94
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
module_function :atom_author
|
|
116
|
-
private_class_method :atom_author
|
|
117
|
-
|
|
118
|
-
def absolute_url(raw, base_url:)
|
|
119
|
-
value = present(raw)
|
|
120
|
-
return unless value
|
|
95
|
+
present(author.name&.content) || present(author.email&.content)
|
|
96
|
+
end
|
|
121
97
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
98
|
+
def absolute_url(raw, base_url:)
|
|
99
|
+
value = present(raw)
|
|
100
|
+
return unless value
|
|
101
|
+
|
|
102
|
+
if base_url
|
|
103
|
+
Html2rss::Url.from_relative(value, base_url)
|
|
104
|
+
else
|
|
105
|
+
Html2rss::Url.from_absolute(value)
|
|
106
|
+
end
|
|
107
|
+
rescue Html2rss::Url::InvalidUrlError
|
|
108
|
+
nil
|
|
126
109
|
end
|
|
127
|
-
rescue ArgumentError
|
|
128
|
-
nil
|
|
129
|
-
end
|
|
130
|
-
module_function :absolute_url
|
|
131
|
-
private_class_method :absolute_url
|
|
132
110
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
111
|
+
def present(value)
|
|
112
|
+
text = value.to_s.strip
|
|
113
|
+
text unless text.empty?
|
|
114
|
+
end
|
|
136
115
|
end
|
|
137
|
-
module_function :present
|
|
138
|
-
private_class_method :present
|
|
139
116
|
end
|
|
140
117
|
end
|
|
141
118
|
end
|
|
@@ -6,8 +6,6 @@ module Html2rss
|
|
|
6
6
|
# Probes list-card +enhance+ value on cached listing HTML.
|
|
7
7
|
# Sole owner of enhance_gains metrics and enhance-specific warn-only warnings.
|
|
8
8
|
module EnhanceAudit # rubocop:disable Metrics/ModuleLength -- probe + compare + heuristics stay co-located
|
|
9
|
-
module_function
|
|
10
|
-
|
|
11
9
|
TRACKED_KEYS = %i[title url description published_at categories author image id enclosures].freeze
|
|
12
10
|
CURATOR_KEYS = %i[title url description published_at].freeze
|
|
13
11
|
private_constant :TRACKED_KEYS, :CURATOR_KEYS
|
|
@@ -28,7 +26,7 @@ module Html2rss
|
|
|
28
26
|
# @param selectors [Hash{Symbol => Object}]
|
|
29
27
|
# @param time_zone [String]
|
|
30
28
|
# @return [AuditSlice, nil] nil when selectors are absent
|
|
31
|
-
def probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
|
|
29
|
+
def self.probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
|
|
32
30
|
return nil unless selectors&.dig(:items, :selector)
|
|
33
31
|
|
|
34
32
|
selector_engine = Selectors.new(response, selectors:, time_zone:)
|
|
@@ -64,7 +62,7 @@ module Html2rss
|
|
|
64
62
|
# @param selectors [Hash{Symbol => Object}]
|
|
65
63
|
# @param time_zone [String]
|
|
66
64
|
# @return [Hash{Symbol => Object}, nil]
|
|
67
|
-
def compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
|
|
65
|
+
def self.compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
|
|
68
66
|
return nil unless selectors&.dig(:items, :selector)
|
|
69
67
|
|
|
70
68
|
off = extraction_stats(response, selectors, time_zone, enhance: false)
|
|
@@ -84,7 +82,7 @@ module Html2rss
|
|
|
84
82
|
|
|
85
83
|
##
|
|
86
84
|
# @return [EnhanceGains] zeroed gains for compare fallback
|
|
87
|
-
def empty_gains
|
|
85
|
+
def self.empty_gains
|
|
88
86
|
EnhanceGains.new(
|
|
89
87
|
items_probed: 0,
|
|
90
88
|
keys_added: zero_keys_added,
|
|
@@ -93,12 +91,12 @@ module Html2rss
|
|
|
93
91
|
)
|
|
94
92
|
end
|
|
95
93
|
|
|
96
|
-
def zero_keys_added
|
|
94
|
+
def self.zero_keys_added
|
|
97
95
|
TRACKED_KEYS.to_h { |key| [key, 0] }.freeze
|
|
98
96
|
end
|
|
99
97
|
private_class_method :zero_keys_added
|
|
100
98
|
|
|
101
|
-
def extraction_stats(response, selectors, time_zone, enhance:)
|
|
99
|
+
def self.extraction_stats(response, selectors, time_zone, enhance:)
|
|
102
100
|
merged = selectors_with_enhance(selectors, enhance:)
|
|
103
101
|
articles = Selectors.new(response, selectors: merged, time_zone:).articles
|
|
104
102
|
{
|
|
@@ -108,14 +106,14 @@ module Html2rss
|
|
|
108
106
|
end
|
|
109
107
|
private_class_method :extraction_stats
|
|
110
108
|
|
|
111
|
-
def selectors_with_enhance(selectors, enhance:)
|
|
109
|
+
def self.selectors_with_enhance(selectors, enhance:)
|
|
112
110
|
merged = HashUtil.deep_dup(selectors)
|
|
113
111
|
merged[:items] = merged.fetch(:items, {}).merge(enhance:)
|
|
114
112
|
merged
|
|
115
113
|
end
|
|
116
114
|
private_class_method :selectors_with_enhance
|
|
117
115
|
|
|
118
|
-
def keys_added(baseline, enhanced)
|
|
116
|
+
def self.keys_added(baseline, enhanced)
|
|
119
117
|
enhanced.each_with_object({}) do |(key, value), counts|
|
|
120
118
|
next if value.nil?
|
|
121
119
|
|
|
@@ -127,7 +125,7 @@ module Html2rss
|
|
|
127
125
|
end
|
|
128
126
|
private_class_method :keys_added
|
|
129
127
|
|
|
130
|
-
def merge_keys_added!(gains, item_keys)
|
|
128
|
+
def self.merge_keys_added!(gains, item_keys)
|
|
131
129
|
gains[:items_probed] += 1
|
|
132
130
|
item_keys.each do |key, count|
|
|
133
131
|
next unless TRACKED_KEYS.include?(key)
|
|
@@ -138,13 +136,13 @@ module Html2rss
|
|
|
138
136
|
end
|
|
139
137
|
private_class_method :merge_keys_added!
|
|
140
138
|
|
|
141
|
-
def finalize_gains!(gains)
|
|
139
|
+
def self.finalize_gains!(gains)
|
|
142
140
|
curator_total = CURATOR_KEYS.sum { |key| gains[:keys_added][key] }
|
|
143
141
|
gains[:no_op] = curator_total.zero?
|
|
144
142
|
end
|
|
145
143
|
private_class_method :finalize_gains!
|
|
146
144
|
|
|
147
|
-
def append_item_warnings!(warnings, item:, enhanced:, item_keys:)
|
|
145
|
+
def self.append_item_warnings!(warnings, item:, enhanced:, item_keys:)
|
|
148
146
|
curator_added = item_keys.slice(*CURATOR_KEYS)
|
|
149
147
|
return if curator_added.empty?
|
|
150
148
|
return unless curator_added.key?(:description)
|
|
@@ -160,7 +158,7 @@ module Html2rss
|
|
|
160
158
|
end
|
|
161
159
|
private_class_method :append_item_warnings!
|
|
162
160
|
|
|
163
|
-
def category_only_description?(description, enhanced)
|
|
161
|
+
def self.category_only_description?(description, enhanced)
|
|
164
162
|
normalized = description.downcase
|
|
165
163
|
return true if Html2rss::Html::ArticleRules::Description::TYPE_CHIPS.include?(normalized)
|
|
166
164
|
|
|
@@ -169,7 +167,7 @@ module Html2rss
|
|
|
169
167
|
end
|
|
170
168
|
private_class_method :category_only_description?
|
|
171
169
|
|
|
172
|
-
def image_only_description?(description, item)
|
|
170
|
+
def self.image_only_description?(description, item)
|
|
173
171
|
images = item.css('img')
|
|
174
172
|
return description == images.first['alt'].to_s.strip if images.size == 1 && images.first['alt']
|
|
175
173
|
|
|
@@ -177,7 +175,7 @@ module Html2rss
|
|
|
177
175
|
end
|
|
178
176
|
private_class_method :image_only_description?
|
|
179
177
|
|
|
180
|
-
def prose_shaped?(text)
|
|
178
|
+
def self.prose_shaped?(text)
|
|
181
179
|
return false if text.match?(IMAGE_FILENAME)
|
|
182
180
|
return false if text.match?(%r{\Ahttps?://}i)
|
|
183
181
|
return false if text.match?(%r{\A/[\w./-]+\z})
|
data/lib/html2rss/test/policy.rb
CHANGED
|
@@ -6,8 +6,6 @@ module Html2rss
|
|
|
6
6
|
# Pass/fail classification for a configuration test after live extraction.
|
|
7
7
|
# Owns min_items vs quality thresholds; {Test.call} orchestrates only.
|
|
8
8
|
module Policy
|
|
9
|
-
module_function
|
|
10
|
-
|
|
11
9
|
Outcome = Data.define(:passed, :failure_kind, :error_message)
|
|
12
10
|
|
|
13
11
|
##
|
|
@@ -28,7 +26,7 @@ module Html2rss
|
|
|
28
26
|
# @param strict_quality [Boolean]
|
|
29
27
|
# @param quality_report [QualityReport]
|
|
30
28
|
# @return [Outcome]
|
|
31
|
-
def evaluate(item_count:, min_items:, strict_quality:, quality_report:)
|
|
29
|
+
def self.evaluate(item_count:, min_items:, strict_quality:, quality_report:)
|
|
32
30
|
min_items_passed = item_count >= min_items
|
|
33
31
|
quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
|
|
34
32
|
passed = min_items_passed && !quality_failed
|
|
@@ -39,7 +37,7 @@ module Html2rss
|
|
|
39
37
|
Outcome.new(passed:, failure_kind:, error_message:)
|
|
40
38
|
end
|
|
41
39
|
|
|
42
|
-
def log_strict_quality_failure(failure_kind, item_count)
|
|
40
|
+
def self.log_strict_quality_failure(failure_kind, item_count)
|
|
43
41
|
Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
|
|
44
42
|
end
|
|
45
43
|
private_class_method :log_strict_quality_failure
|
|
@@ -47,7 +45,7 @@ module Html2rss
|
|
|
47
45
|
##
|
|
48
46
|
# @param quality_report [QualityReport]
|
|
49
47
|
# @return [Boolean]
|
|
50
|
-
def quality_failure?(quality_report)
|
|
48
|
+
def self.quality_failure?(quality_report)
|
|
51
49
|
metrics = quality_report.metrics
|
|
52
50
|
item_count = metrics[:item_count]
|
|
53
51
|
return false if item_count.zero?
|
|
@@ -62,7 +60,7 @@ module Html2rss
|
|
|
62
60
|
# @param min_items [Integer]
|
|
63
61
|
# @param quality_report [QualityReport]
|
|
64
62
|
# @return [Array(FailureKind, nil), Array(String, nil)]
|
|
65
|
-
def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
|
|
63
|
+
def self.outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
|
|
66
64
|
return [nil, nil] if min_items_passed && !quality_failed
|
|
67
65
|
unless min_items_passed
|
|
68
66
|
return [FailureKind[:min_items],
|
|
@@ -72,13 +70,13 @@ module Html2rss
|
|
|
72
70
|
[FailureKind[:quality], quality_failure_message(quality_report)]
|
|
73
71
|
end
|
|
74
72
|
|
|
75
|
-
def quality_failure_message(quality_report)
|
|
73
|
+
def self.quality_failure_message(quality_report)
|
|
76
74
|
reasons = quality_failure_reasons(quality_report.metrics)
|
|
77
75
|
"Feed quality check failed (#{reasons.join(', ')})"
|
|
78
76
|
end
|
|
79
77
|
private_class_method :quality_failure_message
|
|
80
78
|
|
|
81
|
-
def quality_failure_reasons(metrics)
|
|
79
|
+
def self.quality_failure_reasons(metrics)
|
|
82
80
|
reasons = []
|
|
83
81
|
reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
|
|
84
82
|
reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
|
|
@@ -87,17 +85,17 @@ module Html2rss
|
|
|
87
85
|
end
|
|
88
86
|
private_class_method :quality_failure_reasons
|
|
89
87
|
|
|
90
|
-
def duplicate_urls_failure?(metrics)
|
|
88
|
+
def self.duplicate_urls_failure?(metrics)
|
|
91
89
|
metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
|
|
92
90
|
end
|
|
93
91
|
private_class_method :duplicate_urls_failure?
|
|
94
92
|
|
|
95
|
-
def junk_title_ratio_failure?(metrics)
|
|
93
|
+
def self.junk_title_ratio_failure?(metrics)
|
|
96
94
|
metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
|
|
97
95
|
end
|
|
98
96
|
private_class_method :junk_title_ratio_failure?
|
|
99
97
|
|
|
100
|
-
def short_title_failure?(metrics)
|
|
98
|
+
def self.short_title_failure?(metrics)
|
|
101
99
|
metrics[:short_title_count].positive?
|
|
102
100
|
end
|
|
103
101
|
private_class_method :short_title_failure?
|
data/lib/html2rss/test.rb
CHANGED
|
@@ -5,8 +5,6 @@ module Html2rss
|
|
|
5
5
|
# Service that runs schema validation and live feed extraction on a configuration,
|
|
6
6
|
# enforcing minimum item thresholds and capturing sample output.
|
|
7
7
|
module Test # rubocop:disable Metrics/ModuleLength -- Result + FailureKind nest with owner
|
|
8
|
-
module_function
|
|
9
|
-
|
|
10
8
|
FailureKind = Data.define(:name)
|
|
11
9
|
|
|
12
10
|
##
|
|
@@ -163,27 +161,27 @@ module Html2rss
|
|
|
163
161
|
# @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
|
|
164
162
|
# @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
|
|
165
163
|
# @return [Html2rss::Test::Result]
|
|
166
|
-
def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
167
|
-
|
|
164
|
+
def self.call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
165
|
+
strict_quality: false, compare_enhance: false)
|
|
168
166
|
raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
|
|
169
167
|
return validation_failure_result(validation, raw_config) unless validation.success?
|
|
170
168
|
|
|
171
169
|
config = build_test_config(raw_config, strategy, params)
|
|
172
170
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
173
171
|
execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
|
|
174
|
-
rescue
|
|
172
|
+
rescue Html2rss::Error => error
|
|
175
173
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
176
174
|
execution_failure_result(error, raw_config, duration)
|
|
177
175
|
end
|
|
178
176
|
|
|
179
|
-
def build_test_config(raw_config, strategy, params)
|
|
177
|
+
def self.build_test_config(raw_config, strategy, params)
|
|
180
178
|
raw_config[:strategy] = strategy.to_sym if strategy
|
|
181
179
|
raw_config[:params] = params if params&.any?
|
|
182
180
|
Config.from_hash(raw_config)
|
|
183
181
|
end
|
|
184
182
|
private_class_method :build_test_config
|
|
185
183
|
|
|
186
|
-
def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
184
|
+
def self.compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
187
185
|
quality_report_for(
|
|
188
186
|
feed_result.to_rss.items,
|
|
189
187
|
channel_url: raw_config.dig(:channel, :url).to_s,
|
|
@@ -194,20 +192,20 @@ module Html2rss
|
|
|
194
192
|
end
|
|
195
193
|
private_class_method :compile_quality_report
|
|
196
194
|
|
|
197
|
-
def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
|
|
198
|
-
|
|
195
|
+
def self.execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
|
|
196
|
+
compare_enhance:)
|
|
199
197
|
feed_result, pipeline_outcome = extract_feed(raw_config)
|
|
200
198
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
201
199
|
quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
202
200
|
enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
|
|
203
201
|
|
|
204
202
|
build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
|
|
205
|
-
min_items:, strict_quality:, response_body: pipeline_outcome
|
|
203
|
+
min_items:, strict_quality:, response_body: pipeline_outcome.response.body)
|
|
206
204
|
end
|
|
207
205
|
private_class_method :execute_timed_pipeline
|
|
208
206
|
|
|
209
|
-
def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
210
|
-
|
|
207
|
+
def self.build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
208
|
+
min_items:, strict_quality:, response_body:)
|
|
211
209
|
rss_doc = feed_result.to_rss
|
|
212
210
|
outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
|
|
213
211
|
strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
|
|
@@ -232,27 +230,14 @@ module Html2rss
|
|
|
232
230
|
end
|
|
233
231
|
private_class_method :build_test_result
|
|
234
232
|
|
|
235
|
-
def extract_samples(items, limit: 3)
|
|
233
|
+
def self.extract_samples(items, limit: 3)
|
|
236
234
|
items.first(limit).map do |item|
|
|
237
|
-
article =
|
|
238
|
-
{
|
|
239
|
-
title: article.title.to_s.strip,
|
|
240
|
-
url: article.url.to_s,
|
|
241
|
-
published_at: article.published_at
|
|
242
|
-
}
|
|
235
|
+
article = Article.new(title: item.title, url: item.link, published_at: item.pubDate)
|
|
236
|
+
{ title: article.title.to_s.strip, url: article.url.to_s, published_at: article.published_at }
|
|
243
237
|
end
|
|
244
238
|
end
|
|
245
239
|
private_class_method :extract_samples
|
|
246
240
|
|
|
247
|
-
def feed_item_as_article(item)
|
|
248
|
-
case item
|
|
249
|
-
when Article then item
|
|
250
|
-
else
|
|
251
|
-
Article.new(title: item.title, url: item.link, published_at: item.pubDate)
|
|
252
|
-
end
|
|
253
|
-
end
|
|
254
|
-
private_class_method :feed_item_as_article
|
|
255
|
-
|
|
256
241
|
##
|
|
257
242
|
# Builds a ship-quality report for RSS items (shared by test and MCP apply).
|
|
258
243
|
#
|
|
@@ -263,8 +248,8 @@ module Html2rss
|
|
|
263
248
|
# @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
|
|
264
249
|
# @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
|
|
265
250
|
# @return [QualityReport]
|
|
266
|
-
def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
|
|
267
|
-
|
|
251
|
+
def self.quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
|
|
252
|
+
probe_native_feed: true)
|
|
268
253
|
audit = AutoSource::Cleanup.audit_feed_items(items)
|
|
269
254
|
native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
|
|
270
255
|
report = QualityReport.from_audit(audit, native_feed:)
|
|
@@ -272,21 +257,20 @@ module Html2rss
|
|
|
272
257
|
merge_enhance_audit(report, config, pipeline_outcome)
|
|
273
258
|
end
|
|
274
259
|
|
|
275
|
-
def extract_feed(raw_config)
|
|
260
|
+
def self.extract_feed(raw_config)
|
|
276
261
|
outcome, feed_result = FeedPipeline.new(raw_config).to_outcome_and_result
|
|
277
262
|
[feed_result, outcome]
|
|
278
263
|
end
|
|
279
264
|
private_class_method :extract_feed
|
|
280
265
|
|
|
281
|
-
def enhance_enabled?(config)
|
|
266
|
+
def self.enhance_enabled?(config)
|
|
282
267
|
return false unless config.selectors
|
|
283
268
|
|
|
284
269
|
!!config.selectors.dig(:items, :enhance)
|
|
285
270
|
end
|
|
286
271
|
private_class_method :enhance_enabled?
|
|
287
272
|
|
|
288
|
-
def build_enhance_compare(config, pipeline_outcome)
|
|
289
|
-
return nil unless pipeline_outcome
|
|
273
|
+
def self.build_enhance_compare(config, pipeline_outcome)
|
|
290
274
|
return nil unless config.selectors
|
|
291
275
|
|
|
292
276
|
EnhanceAudit.compare(
|
|
@@ -297,9 +281,8 @@ module Html2rss
|
|
|
297
281
|
end
|
|
298
282
|
private_class_method :build_enhance_compare
|
|
299
283
|
|
|
300
|
-
def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
|
|
301
|
-
return report unless
|
|
302
|
-
return report unless config.selectors
|
|
284
|
+
def self.merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
|
|
285
|
+
return report unless enhance_enabled?(config)
|
|
303
286
|
|
|
304
287
|
slice = EnhanceAudit.probe(
|
|
305
288
|
response: pipeline_outcome.response,
|
|
@@ -320,7 +303,7 @@ module Html2rss
|
|
|
320
303
|
end
|
|
321
304
|
private_class_method :merge_enhance_audit
|
|
322
305
|
|
|
323
|
-
def enhance_gains_to_h(enhance_gains)
|
|
306
|
+
def self.enhance_gains_to_h(enhance_gains)
|
|
324
307
|
{
|
|
325
308
|
items_probed: enhance_gains.items_probed,
|
|
326
309
|
keys_added: enhance_gains.keys_added,
|
|
@@ -330,7 +313,7 @@ module Html2rss
|
|
|
330
313
|
end
|
|
331
314
|
private_class_method :enhance_gains_to_h
|
|
332
315
|
|
|
333
|
-
def append_url_mismatch_warning(report, channel_url, feed_result)
|
|
316
|
+
def self.append_url_mismatch_warning(report, channel_url, feed_result)
|
|
334
317
|
return report unless url_mismatch?(channel_url, feed_result)
|
|
335
318
|
|
|
336
319
|
warnings = report.warnings.dup
|
|
@@ -344,22 +327,20 @@ module Html2rss
|
|
|
344
327
|
end
|
|
345
328
|
private_class_method :append_url_mismatch_warning
|
|
346
329
|
|
|
347
|
-
def url_mismatch?(channel_url, feed_result)
|
|
348
|
-
status = feed_result
|
|
349
|
-
return false unless status
|
|
350
|
-
|
|
330
|
+
def self.url_mismatch?(channel_url, feed_result)
|
|
331
|
+
status = feed_result.status
|
|
351
332
|
configured = channel_url.to_s
|
|
352
333
|
final = status.scrape_url.to_s
|
|
353
334
|
final = status.entry_url.to_s if final.empty?
|
|
354
335
|
return false if configured.empty? || final.empty?
|
|
355
336
|
|
|
356
337
|
!Url.from_absolute(configured).same_document?(Url.from_absolute(final))
|
|
357
|
-
rescue
|
|
338
|
+
rescue Html2rss::Url::InvalidUrlError
|
|
358
339
|
false
|
|
359
340
|
end
|
|
360
341
|
private_class_method :url_mismatch?
|
|
361
342
|
|
|
362
|
-
def probe_native_feed_url(channel_url, config)
|
|
343
|
+
def self.probe_native_feed_url(channel_url, config)
|
|
363
344
|
return nil if channel_url.to_s.empty?
|
|
364
345
|
|
|
365
346
|
resources = FeedPipeline::RuntimePolicy.resources_for(config)
|
|
@@ -368,12 +349,12 @@ module Html2rss
|
|
|
368
349
|
config:, strategy:, budget: resources.budget, policy: resources.policy
|
|
369
350
|
)
|
|
370
351
|
Syndication::Discovery.best_feed_url(page_url: channel_url, request_session: session)
|
|
371
|
-
rescue
|
|
352
|
+
rescue Html2rss::Error
|
|
372
353
|
nil
|
|
373
354
|
end
|
|
374
355
|
private_class_method :probe_native_feed_url
|
|
375
356
|
|
|
376
|
-
def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
|
|
357
|
+
def self.validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
|
|
377
358
|
Result.new(
|
|
378
359
|
success: false,
|
|
379
360
|
item_count: 0,
|
|
@@ -391,7 +372,7 @@ module Html2rss
|
|
|
391
372
|
end
|
|
392
373
|
private_class_method :validation_failure_result
|
|
393
374
|
|
|
394
|
-
def execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
|
|
375
|
+
def self.execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
|
|
395
376
|
Result.new(
|
|
396
377
|
success: false,
|
|
397
378
|
item_count: 0,
|