html2rss 0.32.3 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/html2rss/article/enclosure.rb +2 -6
- data/lib/html2rss/auto_source/cleanup.rb +1 -3
- data/lib/html2rss/auto_source/scraper/html.rb +2 -8
- data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
- data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
- data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
- data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
- data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
- data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
- data/lib/html2rss/auto_source/scraper.rb +0 -11
- data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
- data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
- data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
- data/lib/html2rss/auto_source.rb +1 -1
- data/lib/html2rss/batch.rb +22 -32
- data/lib/html2rss/capture/README.md +1 -1
- data/lib/html2rss/capture.rb +7 -27
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +33 -27
- data/lib/html2rss/config/request_controls.rb +10 -2
- data/lib/html2rss/config/schema.rb +8 -10
- data/lib/html2rss/config/selectors_validator.rb +1 -0
- data/lib/html2rss/config/validator.rb +3 -3
- data/lib/html2rss/config.rb +1 -1
- data/lib/html2rss/defaults.rb +5 -16
- data/lib/html2rss/doctor/botasaurus.rb +11 -19
- data/lib/html2rss/error.rb +54 -1
- data/lib/html2rss/feed_builder/rss.rb +1 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +58 -55
- data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +3 -3
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +5 -9
- data/lib/html2rss/feed_result.rb +4 -3
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
- data/lib/html2rss/html/probe.rb +51 -51
- data/lib/html2rss/link_destination/path_classifier.rb +11 -33
- data/lib/html2rss/mcp/config_argument.rb +9 -2
- data/lib/html2rss/mcp/contract.rb +7 -5
- data/lib/html2rss/mcp/outcome.rb +4 -5
- data/lib/html2rss/mcp/runtime.rb +17 -24
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/diagnostics.rb +7 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +10 -12
- data/lib/html2rss/request_service/blocked_surface.rb +3 -7
- data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +1 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +3 -1
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +8 -26
- data/lib/html2rss/request_service/response.rb +3 -6
- data/lib/html2rss/request_service/strategy.rb +2 -2
- data/lib/html2rss/request_service.rb +26 -2
- data/lib/html2rss/request_session/pager/base.rb +3 -3
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/scoring/engine.rb +4 -32
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +14 -18
- data/lib/html2rss/sst/node.rb +9 -3
- data/lib/html2rss/sst/normalizer.rb +25 -24
- data/lib/html2rss/sst/text.rb +53 -57
- data/lib/html2rss/status.rb +23 -30
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/discovery.rb +117 -187
- data/lib/html2rss/syndication/parser.rb +92 -115
- data/lib/html2rss/test/enhance_audit.rb +13 -15
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +29 -48
- data/lib/html2rss/url.rb +53 -53
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +2 -1
- metadata +4 -2
|
@@ -6,8 +6,6 @@ module Html2rss
|
|
|
6
6
|
# Probes list-card +enhance+ value on cached listing HTML.
|
|
7
7
|
# Sole owner of enhance_gains metrics and enhance-specific warn-only warnings.
|
|
8
8
|
module EnhanceAudit # rubocop:disable Metrics/ModuleLength -- probe + compare + heuristics stay co-located
|
|
9
|
-
module_function
|
|
10
|
-
|
|
11
9
|
TRACKED_KEYS = %i[title url description published_at categories author image id enclosures].freeze
|
|
12
10
|
CURATOR_KEYS = %i[title url description published_at].freeze
|
|
13
11
|
private_constant :TRACKED_KEYS, :CURATOR_KEYS
|
|
@@ -28,7 +26,7 @@ module Html2rss
|
|
|
28
26
|
# @param selectors [Hash{Symbol => Object}]
|
|
29
27
|
# @param time_zone [String]
|
|
30
28
|
# @return [AuditSlice, nil] nil when selectors are absent
|
|
31
|
-
def probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
|
|
29
|
+
def self.probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
|
|
32
30
|
return nil unless selectors&.dig(:items, :selector)
|
|
33
31
|
|
|
34
32
|
selector_engine = Selectors.new(response, selectors:, time_zone:)
|
|
@@ -64,7 +62,7 @@ module Html2rss
|
|
|
64
62
|
# @param selectors [Hash{Symbol => Object}]
|
|
65
63
|
# @param time_zone [String]
|
|
66
64
|
# @return [Hash{Symbol => Object}, nil]
|
|
67
|
-
def compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
|
|
65
|
+
def self.compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
|
|
68
66
|
return nil unless selectors&.dig(:items, :selector)
|
|
69
67
|
|
|
70
68
|
off = extraction_stats(response, selectors, time_zone, enhance: false)
|
|
@@ -84,7 +82,7 @@ module Html2rss
|
|
|
84
82
|
|
|
85
83
|
##
|
|
86
84
|
# @return [EnhanceGains] zeroed gains for compare fallback
|
|
87
|
-
def empty_gains
|
|
85
|
+
def self.empty_gains
|
|
88
86
|
EnhanceGains.new(
|
|
89
87
|
items_probed: 0,
|
|
90
88
|
keys_added: zero_keys_added,
|
|
@@ -93,12 +91,12 @@ module Html2rss
|
|
|
93
91
|
)
|
|
94
92
|
end
|
|
95
93
|
|
|
96
|
-
def zero_keys_added
|
|
94
|
+
def self.zero_keys_added
|
|
97
95
|
TRACKED_KEYS.to_h { |key| [key, 0] }.freeze
|
|
98
96
|
end
|
|
99
97
|
private_class_method :zero_keys_added
|
|
100
98
|
|
|
101
|
-
def extraction_stats(response, selectors, time_zone, enhance:)
|
|
99
|
+
def self.extraction_stats(response, selectors, time_zone, enhance:)
|
|
102
100
|
merged = selectors_with_enhance(selectors, enhance:)
|
|
103
101
|
articles = Selectors.new(response, selectors: merged, time_zone:).articles
|
|
104
102
|
{
|
|
@@ -108,14 +106,14 @@ module Html2rss
|
|
|
108
106
|
end
|
|
109
107
|
private_class_method :extraction_stats
|
|
110
108
|
|
|
111
|
-
def selectors_with_enhance(selectors, enhance:)
|
|
109
|
+
def self.selectors_with_enhance(selectors, enhance:)
|
|
112
110
|
merged = HashUtil.deep_dup(selectors)
|
|
113
111
|
merged[:items] = merged.fetch(:items, {}).merge(enhance:)
|
|
114
112
|
merged
|
|
115
113
|
end
|
|
116
114
|
private_class_method :selectors_with_enhance
|
|
117
115
|
|
|
118
|
-
def keys_added(baseline, enhanced)
|
|
116
|
+
def self.keys_added(baseline, enhanced)
|
|
119
117
|
enhanced.each_with_object({}) do |(key, value), counts|
|
|
120
118
|
next if value.nil?
|
|
121
119
|
|
|
@@ -127,7 +125,7 @@ module Html2rss
|
|
|
127
125
|
end
|
|
128
126
|
private_class_method :keys_added
|
|
129
127
|
|
|
130
|
-
def merge_keys_added!(gains, item_keys)
|
|
128
|
+
def self.merge_keys_added!(gains, item_keys)
|
|
131
129
|
gains[:items_probed] += 1
|
|
132
130
|
item_keys.each do |key, count|
|
|
133
131
|
next unless TRACKED_KEYS.include?(key)
|
|
@@ -138,13 +136,13 @@ module Html2rss
|
|
|
138
136
|
end
|
|
139
137
|
private_class_method :merge_keys_added!
|
|
140
138
|
|
|
141
|
-
def finalize_gains!(gains)
|
|
139
|
+
def self.finalize_gains!(gains)
|
|
142
140
|
curator_total = CURATOR_KEYS.sum { |key| gains[:keys_added][key] }
|
|
143
141
|
gains[:no_op] = curator_total.zero?
|
|
144
142
|
end
|
|
145
143
|
private_class_method :finalize_gains!
|
|
146
144
|
|
|
147
|
-
def append_item_warnings!(warnings, item:, enhanced:, item_keys:)
|
|
145
|
+
def self.append_item_warnings!(warnings, item:, enhanced:, item_keys:)
|
|
148
146
|
curator_added = item_keys.slice(*CURATOR_KEYS)
|
|
149
147
|
return if curator_added.empty?
|
|
150
148
|
return unless curator_added.key?(:description)
|
|
@@ -160,7 +158,7 @@ module Html2rss
|
|
|
160
158
|
end
|
|
161
159
|
private_class_method :append_item_warnings!
|
|
162
160
|
|
|
163
|
-
def category_only_description?(description, enhanced)
|
|
161
|
+
def self.category_only_description?(description, enhanced)
|
|
164
162
|
normalized = description.downcase
|
|
165
163
|
return true if Html2rss::Html::ArticleRules::Description::TYPE_CHIPS.include?(normalized)
|
|
166
164
|
|
|
@@ -169,7 +167,7 @@ module Html2rss
|
|
|
169
167
|
end
|
|
170
168
|
private_class_method :category_only_description?
|
|
171
169
|
|
|
172
|
-
def image_only_description?(description, item)
|
|
170
|
+
def self.image_only_description?(description, item)
|
|
173
171
|
images = item.css('img')
|
|
174
172
|
return description == images.first['alt'].to_s.strip if images.size == 1 && images.first['alt']
|
|
175
173
|
|
|
@@ -177,7 +175,7 @@ module Html2rss
|
|
|
177
175
|
end
|
|
178
176
|
private_class_method :image_only_description?
|
|
179
177
|
|
|
180
|
-
def prose_shaped?(text)
|
|
178
|
+
def self.prose_shaped?(text)
|
|
181
179
|
return false if text.match?(IMAGE_FILENAME)
|
|
182
180
|
return false if text.match?(%r{\Ahttps?://}i)
|
|
183
181
|
return false if text.match?(%r{\A/[\w./-]+\z})
|
data/lib/html2rss/test/policy.rb
CHANGED
|
@@ -6,8 +6,6 @@ module Html2rss
|
|
|
6
6
|
# Pass/fail classification for a configuration test after live extraction.
|
|
7
7
|
# Owns min_items vs quality thresholds; {Test.call} orchestrates only.
|
|
8
8
|
module Policy
|
|
9
|
-
module_function
|
|
10
|
-
|
|
11
9
|
Outcome = Data.define(:passed, :failure_kind, :error_message)
|
|
12
10
|
|
|
13
11
|
##
|
|
@@ -28,7 +26,7 @@ module Html2rss
|
|
|
28
26
|
# @param strict_quality [Boolean]
|
|
29
27
|
# @param quality_report [QualityReport]
|
|
30
28
|
# @return [Outcome]
|
|
31
|
-
def evaluate(item_count:, min_items:, strict_quality:, quality_report:)
|
|
29
|
+
def self.evaluate(item_count:, min_items:, strict_quality:, quality_report:)
|
|
32
30
|
min_items_passed = item_count >= min_items
|
|
33
31
|
quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
|
|
34
32
|
passed = min_items_passed && !quality_failed
|
|
@@ -39,7 +37,7 @@ module Html2rss
|
|
|
39
37
|
Outcome.new(passed:, failure_kind:, error_message:)
|
|
40
38
|
end
|
|
41
39
|
|
|
42
|
-
def log_strict_quality_failure(failure_kind, item_count)
|
|
40
|
+
def self.log_strict_quality_failure(failure_kind, item_count)
|
|
43
41
|
Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
|
|
44
42
|
end
|
|
45
43
|
private_class_method :log_strict_quality_failure
|
|
@@ -47,7 +45,7 @@ module Html2rss
|
|
|
47
45
|
##
|
|
48
46
|
# @param quality_report [QualityReport]
|
|
49
47
|
# @return [Boolean]
|
|
50
|
-
def quality_failure?(quality_report)
|
|
48
|
+
def self.quality_failure?(quality_report)
|
|
51
49
|
metrics = quality_report.metrics
|
|
52
50
|
item_count = metrics[:item_count]
|
|
53
51
|
return false if item_count.zero?
|
|
@@ -62,7 +60,7 @@ module Html2rss
|
|
|
62
60
|
# @param min_items [Integer]
|
|
63
61
|
# @param quality_report [QualityReport]
|
|
64
62
|
# @return [Array(FailureKind, nil), Array(String, nil)]
|
|
65
|
-
def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
|
|
63
|
+
def self.outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
|
|
66
64
|
return [nil, nil] if min_items_passed && !quality_failed
|
|
67
65
|
unless min_items_passed
|
|
68
66
|
return [FailureKind[:min_items],
|
|
@@ -72,13 +70,13 @@ module Html2rss
|
|
|
72
70
|
[FailureKind[:quality], quality_failure_message(quality_report)]
|
|
73
71
|
end
|
|
74
72
|
|
|
75
|
-
def quality_failure_message(quality_report)
|
|
73
|
+
def self.quality_failure_message(quality_report)
|
|
76
74
|
reasons = quality_failure_reasons(quality_report.metrics)
|
|
77
75
|
"Feed quality check failed (#{reasons.join(', ')})"
|
|
78
76
|
end
|
|
79
77
|
private_class_method :quality_failure_message
|
|
80
78
|
|
|
81
|
-
def quality_failure_reasons(metrics)
|
|
79
|
+
def self.quality_failure_reasons(metrics)
|
|
82
80
|
reasons = []
|
|
83
81
|
reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
|
|
84
82
|
reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
|
|
@@ -87,17 +85,17 @@ module Html2rss
|
|
|
87
85
|
end
|
|
88
86
|
private_class_method :quality_failure_reasons
|
|
89
87
|
|
|
90
|
-
def duplicate_urls_failure?(metrics)
|
|
88
|
+
def self.duplicate_urls_failure?(metrics)
|
|
91
89
|
metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
|
|
92
90
|
end
|
|
93
91
|
private_class_method :duplicate_urls_failure?
|
|
94
92
|
|
|
95
|
-
def junk_title_ratio_failure?(metrics)
|
|
93
|
+
def self.junk_title_ratio_failure?(metrics)
|
|
96
94
|
metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
|
|
97
95
|
end
|
|
98
96
|
private_class_method :junk_title_ratio_failure?
|
|
99
97
|
|
|
100
|
-
def short_title_failure?(metrics)
|
|
98
|
+
def self.short_title_failure?(metrics)
|
|
101
99
|
metrics[:short_title_count].positive?
|
|
102
100
|
end
|
|
103
101
|
private_class_method :short_title_failure?
|
data/lib/html2rss/test.rb
CHANGED
|
@@ -5,8 +5,6 @@ module Html2rss
|
|
|
5
5
|
# Service that runs schema validation and live feed extraction on a configuration,
|
|
6
6
|
# enforcing minimum item thresholds and capturing sample output.
|
|
7
7
|
module Test # rubocop:disable Metrics/ModuleLength -- Result + FailureKind nest with owner
|
|
8
|
-
module_function
|
|
9
|
-
|
|
10
8
|
FailureKind = Data.define(:name)
|
|
11
9
|
|
|
12
10
|
##
|
|
@@ -163,27 +161,27 @@ module Html2rss
|
|
|
163
161
|
# @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
|
|
164
162
|
# @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
|
|
165
163
|
# @return [Html2rss::Test::Result]
|
|
166
|
-
def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
167
|
-
|
|
164
|
+
def self.call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
165
|
+
strict_quality: false, compare_enhance: false)
|
|
168
166
|
raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
|
|
169
167
|
return validation_failure_result(validation, raw_config) unless validation.success?
|
|
170
168
|
|
|
171
169
|
config = build_test_config(raw_config, strategy, params)
|
|
172
170
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
173
171
|
execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
|
|
174
|
-
rescue
|
|
172
|
+
rescue Html2rss::Error => error
|
|
175
173
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
176
174
|
execution_failure_result(error, raw_config, duration)
|
|
177
175
|
end
|
|
178
176
|
|
|
179
|
-
def build_test_config(raw_config, strategy, params)
|
|
177
|
+
def self.build_test_config(raw_config, strategy, params)
|
|
180
178
|
raw_config[:strategy] = strategy.to_sym if strategy
|
|
181
179
|
raw_config[:params] = params if params&.any?
|
|
182
180
|
Config.from_hash(raw_config)
|
|
183
181
|
end
|
|
184
182
|
private_class_method :build_test_config
|
|
185
183
|
|
|
186
|
-
def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
184
|
+
def self.compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
187
185
|
quality_report_for(
|
|
188
186
|
feed_result.to_rss.items,
|
|
189
187
|
channel_url: raw_config.dig(:channel, :url).to_s,
|
|
@@ -194,20 +192,20 @@ module Html2rss
|
|
|
194
192
|
end
|
|
195
193
|
private_class_method :compile_quality_report
|
|
196
194
|
|
|
197
|
-
def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
|
|
198
|
-
|
|
195
|
+
def self.execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
|
|
196
|
+
compare_enhance:)
|
|
199
197
|
feed_result, pipeline_outcome = extract_feed(raw_config)
|
|
200
198
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
201
199
|
quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
202
200
|
enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
|
|
203
201
|
|
|
204
202
|
build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
|
|
205
|
-
min_items:, strict_quality:, response_body: pipeline_outcome
|
|
203
|
+
min_items:, strict_quality:, response_body: pipeline_outcome.response.body)
|
|
206
204
|
end
|
|
207
205
|
private_class_method :execute_timed_pipeline
|
|
208
206
|
|
|
209
|
-
def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
210
|
-
|
|
207
|
+
def self.build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
208
|
+
min_items:, strict_quality:, response_body:)
|
|
211
209
|
rss_doc = feed_result.to_rss
|
|
212
210
|
outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
|
|
213
211
|
strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
|
|
@@ -232,27 +230,14 @@ module Html2rss
|
|
|
232
230
|
end
|
|
233
231
|
private_class_method :build_test_result
|
|
234
232
|
|
|
235
|
-
def extract_samples(items, limit: 3)
|
|
233
|
+
def self.extract_samples(items, limit: 3)
|
|
236
234
|
items.first(limit).map do |item|
|
|
237
|
-
article =
|
|
238
|
-
{
|
|
239
|
-
title: article.title.to_s.strip,
|
|
240
|
-
url: article.url.to_s,
|
|
241
|
-
published_at: article.published_at
|
|
242
|
-
}
|
|
235
|
+
article = Article.new(title: item.title, url: item.link, published_at: item.pubDate)
|
|
236
|
+
{ title: article.title.to_s.strip, url: article.url.to_s, published_at: article.published_at }
|
|
243
237
|
end
|
|
244
238
|
end
|
|
245
239
|
private_class_method :extract_samples
|
|
246
240
|
|
|
247
|
-
def feed_item_as_article(item)
|
|
248
|
-
case item
|
|
249
|
-
when Article then item
|
|
250
|
-
else
|
|
251
|
-
Article.new(title: item.title, url: item.link, published_at: item.pubDate)
|
|
252
|
-
end
|
|
253
|
-
end
|
|
254
|
-
private_class_method :feed_item_as_article
|
|
255
|
-
|
|
256
241
|
##
|
|
257
242
|
# Builds a ship-quality report for RSS items (shared by test and MCP apply).
|
|
258
243
|
#
|
|
@@ -263,8 +248,8 @@ module Html2rss
|
|
|
263
248
|
# @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
|
|
264
249
|
# @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
|
|
265
250
|
# @return [QualityReport]
|
|
266
|
-
def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
|
|
267
|
-
|
|
251
|
+
def self.quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
|
|
252
|
+
probe_native_feed: true)
|
|
268
253
|
audit = AutoSource::Cleanup.audit_feed_items(items)
|
|
269
254
|
native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
|
|
270
255
|
report = QualityReport.from_audit(audit, native_feed:)
|
|
@@ -272,21 +257,20 @@ module Html2rss
|
|
|
272
257
|
merge_enhance_audit(report, config, pipeline_outcome)
|
|
273
258
|
end
|
|
274
259
|
|
|
275
|
-
def extract_feed(raw_config)
|
|
260
|
+
def self.extract_feed(raw_config)
|
|
276
261
|
outcome, feed_result = FeedPipeline.new(raw_config).to_outcome_and_result
|
|
277
262
|
[feed_result, outcome]
|
|
278
263
|
end
|
|
279
264
|
private_class_method :extract_feed
|
|
280
265
|
|
|
281
|
-
def enhance_enabled?(config)
|
|
266
|
+
def self.enhance_enabled?(config)
|
|
282
267
|
return false unless config.selectors
|
|
283
268
|
|
|
284
269
|
!!config.selectors.dig(:items, :enhance)
|
|
285
270
|
end
|
|
286
271
|
private_class_method :enhance_enabled?
|
|
287
272
|
|
|
288
|
-
def build_enhance_compare(config, pipeline_outcome)
|
|
289
|
-
return nil unless pipeline_outcome
|
|
273
|
+
def self.build_enhance_compare(config, pipeline_outcome)
|
|
290
274
|
return nil unless config.selectors
|
|
291
275
|
|
|
292
276
|
EnhanceAudit.compare(
|
|
@@ -297,9 +281,8 @@ module Html2rss
|
|
|
297
281
|
end
|
|
298
282
|
private_class_method :build_enhance_compare
|
|
299
283
|
|
|
300
|
-
def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
|
|
301
|
-
return report unless
|
|
302
|
-
return report unless config.selectors
|
|
284
|
+
def self.merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
|
|
285
|
+
return report unless enhance_enabled?(config)
|
|
303
286
|
|
|
304
287
|
slice = EnhanceAudit.probe(
|
|
305
288
|
response: pipeline_outcome.response,
|
|
@@ -320,7 +303,7 @@ module Html2rss
|
|
|
320
303
|
end
|
|
321
304
|
private_class_method :merge_enhance_audit
|
|
322
305
|
|
|
323
|
-
def enhance_gains_to_h(enhance_gains)
|
|
306
|
+
def self.enhance_gains_to_h(enhance_gains)
|
|
324
307
|
{
|
|
325
308
|
items_probed: enhance_gains.items_probed,
|
|
326
309
|
keys_added: enhance_gains.keys_added,
|
|
@@ -330,7 +313,7 @@ module Html2rss
|
|
|
330
313
|
end
|
|
331
314
|
private_class_method :enhance_gains_to_h
|
|
332
315
|
|
|
333
|
-
def append_url_mismatch_warning(report, channel_url, feed_result)
|
|
316
|
+
def self.append_url_mismatch_warning(report, channel_url, feed_result)
|
|
334
317
|
return report unless url_mismatch?(channel_url, feed_result)
|
|
335
318
|
|
|
336
319
|
warnings = report.warnings.dup
|
|
@@ -344,22 +327,20 @@ module Html2rss
|
|
|
344
327
|
end
|
|
345
328
|
private_class_method :append_url_mismatch_warning
|
|
346
329
|
|
|
347
|
-
def url_mismatch?(channel_url, feed_result)
|
|
348
|
-
status = feed_result
|
|
349
|
-
return false unless status
|
|
350
|
-
|
|
330
|
+
def self.url_mismatch?(channel_url, feed_result)
|
|
331
|
+
status = feed_result.status
|
|
351
332
|
configured = channel_url.to_s
|
|
352
333
|
final = status.scrape_url.to_s
|
|
353
334
|
final = status.entry_url.to_s if final.empty?
|
|
354
335
|
return false if configured.empty? || final.empty?
|
|
355
336
|
|
|
356
337
|
!Url.from_absolute(configured).same_document?(Url.from_absolute(final))
|
|
357
|
-
rescue
|
|
338
|
+
rescue Html2rss::Url::InvalidUrlError
|
|
358
339
|
false
|
|
359
340
|
end
|
|
360
341
|
private_class_method :url_mismatch?
|
|
361
342
|
|
|
362
|
-
def probe_native_feed_url(channel_url, config)
|
|
343
|
+
def self.probe_native_feed_url(channel_url, config)
|
|
363
344
|
return nil if channel_url.to_s.empty?
|
|
364
345
|
|
|
365
346
|
resources = FeedPipeline::RuntimePolicy.resources_for(config)
|
|
@@ -368,12 +349,12 @@ module Html2rss
|
|
|
368
349
|
config:, strategy:, budget: resources.budget, policy: resources.policy
|
|
369
350
|
)
|
|
370
351
|
Syndication::Discovery.best_feed_url(page_url: channel_url, request_session: session)
|
|
371
|
-
rescue
|
|
352
|
+
rescue Html2rss::Error
|
|
372
353
|
nil
|
|
373
354
|
end
|
|
374
355
|
private_class_method :probe_native_feed_url
|
|
375
356
|
|
|
376
|
-
def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
|
|
357
|
+
def self.validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
|
|
377
358
|
Result.new(
|
|
378
359
|
success: false,
|
|
379
360
|
item_count: 0,
|
|
@@ -391,7 +372,7 @@ module Html2rss
|
|
|
391
372
|
end
|
|
392
373
|
private_class_method :validation_failure_result
|
|
393
374
|
|
|
394
|
-
def execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
|
|
375
|
+
def self.execution_failure_result(error, raw_config, duration) # rubocop:disable Metrics/MethodLength
|
|
395
376
|
Result.new(
|
|
396
377
|
success: false,
|
|
397
378
|
item_count: 0,
|
data/lib/html2rss/url.rb
CHANGED
|
@@ -26,6 +26,9 @@ module Html2rss
|
|
|
26
26
|
class Url
|
|
27
27
|
include Comparable
|
|
28
28
|
|
|
29
|
+
# Raised when a string cannot be parsed or resolved to a valid URL.
|
|
30
|
+
class InvalidUrlError < Html2rss::Error; end
|
|
31
|
+
|
|
29
32
|
# Regular expression for basic URI format validation
|
|
30
33
|
URI_REGEXP = Addressable::URI::URIREGEX
|
|
31
34
|
# Schemes accepted by channel URL validation.
|
|
@@ -37,22 +40,21 @@ module Html2rss
|
|
|
37
40
|
# @param relative_url [String, Html2rss::Url] the relative URL to resolve
|
|
38
41
|
# @param base_url [String, Html2rss::Url] the base URL to resolve against
|
|
39
42
|
# @return [Url] the resolved absolute URL
|
|
40
|
-
# @raise [
|
|
43
|
+
# @raise [InvalidUrlError] if the URL cannot be parsed
|
|
41
44
|
def self.from_relative(relative_url, base_url)
|
|
42
45
|
url = Addressable::URI.parse(relative_url.to_s.strip)
|
|
43
46
|
return new(url) if url.absolute?
|
|
44
47
|
|
|
45
|
-
|
|
48
|
+
join_parsed_base(base_url, url)
|
|
46
49
|
rescue Addressable::URI::InvalidURIError
|
|
47
|
-
raise
|
|
50
|
+
raise InvalidUrlError, 'URL could not be parsed'
|
|
48
51
|
end
|
|
49
52
|
|
|
50
|
-
# @param base_url [String]
|
|
53
|
+
# @param base_url [String, Url]
|
|
51
54
|
# @param url [Addressable::URI]
|
|
52
55
|
# @return [Url]
|
|
53
56
|
def self.join_parsed_base(base_url, url)
|
|
54
57
|
uri = Addressable::URI.parse(base_url.to_s)
|
|
55
|
-
uri.path = '/' if uri.path.empty?
|
|
56
58
|
new(uri.join(url).normalize)
|
|
57
59
|
end
|
|
58
60
|
private_class_method :join_parsed_base
|
|
@@ -64,10 +66,10 @@ module Html2rss
|
|
|
64
66
|
# @param raw_url [String] the raw URL string to sanitize
|
|
65
67
|
# @return [Url, nil] the sanitized URL, or nil if no valid URL found
|
|
66
68
|
def self.sanitize(raw_url)
|
|
67
|
-
match = raw_url.to_s.match(%r{(?:(?:https?|ftp
|
|
69
|
+
match = raw_url.to_s.match(%r{(?:(?:https?|ftp)://|mailto:)[^\s<>"]+})
|
|
68
70
|
return unless match
|
|
69
71
|
|
|
70
|
-
new(Addressable::URI.parse(match[0]
|
|
72
|
+
new(Addressable::URI.parse(match[0]).normalize)
|
|
71
73
|
end
|
|
72
74
|
|
|
73
75
|
##
|
|
@@ -75,15 +77,15 @@ module Html2rss
|
|
|
75
77
|
#
|
|
76
78
|
# @param url_string [String, Html2rss::Url] the absolute URL to parse
|
|
77
79
|
# @return [Url] the parsed and normalized URL
|
|
78
|
-
# @raise [
|
|
80
|
+
# @raise [InvalidUrlError] if the URL is not absolute or cannot be parsed
|
|
79
81
|
def self.from_absolute(url_string)
|
|
80
82
|
return url_string if url_string.is_a?(self)
|
|
81
83
|
|
|
82
|
-
new(Addressable::URI.parse(url_string.to_s.
|
|
83
|
-
raise
|
|
84
|
+
new(Addressable::URI.parse(url_string.to_s.lstrip).normalize).tap do |url|
|
|
85
|
+
raise InvalidUrlError, 'URL must be absolute' unless url.absolute?
|
|
84
86
|
end
|
|
85
87
|
rescue Addressable::URI::InvalidURIError
|
|
86
|
-
raise
|
|
88
|
+
raise InvalidUrlError, 'URL must be absolute'
|
|
87
89
|
end
|
|
88
90
|
|
|
89
91
|
##
|
|
@@ -92,20 +94,30 @@ module Html2rss
|
|
|
92
94
|
#
|
|
93
95
|
# @param url_string [String] the URL string to validate and parse
|
|
94
96
|
# @return [Url] the validated and parsed URL
|
|
95
|
-
# @raise [
|
|
97
|
+
# @raise [InvalidUrlError] if the URL doesn't meet channel requirements
|
|
96
98
|
# @example Creating a channel URL
|
|
97
99
|
# Url.for_channel('https://example.com')
|
|
98
100
|
# # => #<Html2rss::Url:... @uri=#<Addressable::URI:... URI:https://example.com>>
|
|
99
101
|
# @example Invalid channel URL
|
|
100
102
|
# Url.for_channel('/relative/path')
|
|
101
|
-
# # => raises
|
|
103
|
+
# # => raises InvalidUrlError: "URL must be absolute"
|
|
102
104
|
def self.for_channel(url_string)
|
|
103
|
-
stripped = url_string.to_s.
|
|
105
|
+
stripped = url_string.to_s.lstrip
|
|
104
106
|
return nil if stripped.empty?
|
|
105
107
|
|
|
106
108
|
from_absolute(stripped).tap { validate_channel_url(_1) }
|
|
107
109
|
end
|
|
108
110
|
|
|
111
|
+
##
|
|
112
|
+
# Creates an absolute http(s) URL, for example for a channel image.
|
|
113
|
+
#
|
|
114
|
+
# @param url_string [String, Html2rss::Url] the URL to validate and parse
|
|
115
|
+
# @return [Url] the validated and parsed URL
|
|
116
|
+
# @raise [InvalidUrlError] if the URL is not absolute or not http(s)
|
|
117
|
+
def self.from_absolute_web(url_string)
|
|
118
|
+
from_absolute(url_string).tap { validate_scheme(_1) }
|
|
119
|
+
end
|
|
120
|
+
|
|
109
121
|
##
|
|
110
122
|
# Extracts a base URL from HTML metadata tags.
|
|
111
123
|
#
|
|
@@ -116,10 +128,10 @@ module Html2rss
|
|
|
116
128
|
tags = { 'link[rel="canonical"]' => 'href', 'meta[property="og:url"]' => 'content',
|
|
117
129
|
'meta[name="twitter:url"]' => 'content', 'base[href]' => 'href' }
|
|
118
130
|
tags.each do |sel, attr|
|
|
119
|
-
val = doc.at_css(sel)&.[](attr)
|
|
120
|
-
return from_absolute(val)
|
|
121
|
-
rescue
|
|
122
|
-
|
|
131
|
+
val = doc.at_css(sel)&.[](attr)
|
|
132
|
+
return from_absolute(val)
|
|
133
|
+
rescue InvalidUrlError
|
|
134
|
+
nil
|
|
123
135
|
end
|
|
124
136
|
nil
|
|
125
137
|
end
|
|
@@ -128,31 +140,32 @@ module Html2rss
|
|
|
128
140
|
# Validates that a URL meets channel requirements.
|
|
129
141
|
#
|
|
130
142
|
# @param url [Url] the URL to validate
|
|
131
|
-
# @raise [
|
|
143
|
+
# @raise [InvalidUrlError] if the URL doesn't meet channel requirements
|
|
132
144
|
def self.validate_channel_url(url)
|
|
133
|
-
|
|
145
|
+
has_forbidden_at = url.user
|
|
146
|
+
has_forbidden_at ||= [url.query, url.fragment].compact.any? { |value| value.include?('@') }
|
|
147
|
+
raise InvalidUrlError, 'URL must not contain an @ character' if has_forbidden_at
|
|
134
148
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
has_forbidden_at ||= [uri.query, uri.fragment].compact.any? { |value| value.include?('@') }
|
|
138
|
-
raise ArgumentError, 'URL must not contain an @ character' if has_forbidden_at
|
|
149
|
+
validate_scheme(url)
|
|
150
|
+
end
|
|
139
151
|
|
|
152
|
+
def self.validate_scheme(url)
|
|
140
153
|
scheme = url.scheme
|
|
141
|
-
raise
|
|
154
|
+
raise InvalidUrlError, "URL scheme '#{scheme}' is not supported" unless SUPPORTED_SCHEMES.include?(scheme)
|
|
142
155
|
end
|
|
143
156
|
|
|
144
|
-
private_class_method :validate_channel_url
|
|
157
|
+
private_class_method :validate_channel_url, :validate_scheme
|
|
145
158
|
|
|
146
159
|
##
|
|
147
160
|
# @param uri [Addressable::URI] the underlying Addressable::URI object (internal use only)
|
|
148
161
|
def initialize(uri)
|
|
149
162
|
@uri = uri.freeze
|
|
150
|
-
@path_segments =
|
|
163
|
+
@path_segments = path.split('/').reject(&:empty?).freeze
|
|
151
164
|
freeze
|
|
152
165
|
end
|
|
153
166
|
|
|
154
167
|
# @return [String] normalized URL string
|
|
155
|
-
def to_s = @uri.
|
|
168
|
+
def to_s = @uri.to_str
|
|
156
169
|
|
|
157
170
|
# @return [String, nil] URI scheme, for example `http` or `https`
|
|
158
171
|
def scheme = @uri.scheme
|
|
@@ -160,6 +173,9 @@ module Html2rss
|
|
|
160
173
|
# @return [String, nil] URI host component
|
|
161
174
|
def host = @uri.host
|
|
162
175
|
|
|
176
|
+
# @return [String, nil] userinfo user, without the password
|
|
177
|
+
def user = @uri.user
|
|
178
|
+
|
|
163
179
|
# Registrable domain (eTLD+1). Same-site hosts like www.wp.pl and wiadomosci.wp.pl
|
|
164
180
|
# share `wp.pl`. IP literals stay the host so 1.2.3.4 is not treated as `3.4`.
|
|
165
181
|
#
|
|
@@ -189,7 +205,7 @@ module Html2rss
|
|
|
189
205
|
# Returns the URL query string as a hash of string keys and values.
|
|
190
206
|
#
|
|
191
207
|
# @return [Hash{String => String}] normalized query parameters
|
|
192
|
-
def query_values = @uri.query_values
|
|
208
|
+
def query_values = @uri.query_values || {}
|
|
193
209
|
|
|
194
210
|
##
|
|
195
211
|
# Returns the URL path split into non-empty segments.
|
|
@@ -205,7 +221,7 @@ module Html2rss
|
|
|
205
221
|
def with_path(path)
|
|
206
222
|
uri = @uri.dup
|
|
207
223
|
uri.path = path
|
|
208
|
-
self.class.from_absolute(uri
|
|
224
|
+
self.class.from_absolute(uri)
|
|
209
225
|
end
|
|
210
226
|
|
|
211
227
|
##
|
|
@@ -213,7 +229,7 @@ module Html2rss
|
|
|
213
229
|
#
|
|
214
230
|
# @return [Url] this URL when already fragment-free; otherwise a new URL
|
|
215
231
|
def without_fragment
|
|
216
|
-
|
|
232
|
+
fragment ? self.class.from_absolute(@uri.omit(:fragment)) : self
|
|
217
233
|
end
|
|
218
234
|
|
|
219
235
|
##
|
|
@@ -224,7 +240,7 @@ module Html2rss
|
|
|
224
240
|
def with_query_values(values)
|
|
225
241
|
uri = @uri.dup
|
|
226
242
|
uri.query_values = values.transform_keys(&:to_s).transform_values(&:to_s)
|
|
227
|
-
self.class.from_absolute(uri
|
|
243
|
+
self.class.from_absolute(uri)
|
|
228
244
|
end
|
|
229
245
|
|
|
230
246
|
##
|
|
@@ -240,15 +256,7 @@ module Html2rss
|
|
|
240
256
|
# url = Url.from_absolute('https://example.com/hello%20world/article.html')
|
|
241
257
|
# url.titleized # => "Hello World Article"
|
|
242
258
|
def titleized
|
|
243
|
-
|
|
244
|
-
return '' if path.empty?
|
|
245
|
-
|
|
246
|
-
nicer_path = CGI.unescapeURIComponent(path)
|
|
247
|
-
.split('/')
|
|
248
|
-
.flat_map do |part|
|
|
249
|
-
part.gsub(/[^a-zA-Z0-9.]/, ' ').gsub(/\s+/, ' ').split
|
|
250
|
-
end
|
|
251
|
-
|
|
259
|
+
nicer_path = CGI.unescapeURIComponent(path).gsub(/[^a-zA-Z0-9.]/, ' ').split
|
|
252
260
|
nicer_path.map!(&:capitalize)
|
|
253
261
|
File.basename(nicer_path.join(' '), '.*')
|
|
254
262
|
end
|
|
@@ -266,8 +274,7 @@ module Html2rss
|
|
|
266
274
|
# url = Url.from_absolute('https://example.com')
|
|
267
275
|
# url.channel_titleized # => "example.com"
|
|
268
276
|
def channel_titleized
|
|
269
|
-
nicer_path = CGI.unescapeURIComponent(
|
|
270
|
-
host = @uri.host
|
|
277
|
+
nicer_path = CGI.unescapeURIComponent(path).split('/').reject(&:empty?)
|
|
271
278
|
|
|
272
279
|
nicer_path.any? ? "#{host}: #{nicer_path.map(&:capitalize).join(' ')}" : host
|
|
273
280
|
end
|
|
@@ -318,12 +325,7 @@ module Html2rss
|
|
|
318
325
|
# @param relative_uri [Addressable::URI]
|
|
319
326
|
# @return [Url]
|
|
320
327
|
def join_uri(relative_uri)
|
|
321
|
-
|
|
322
|
-
if base.path.empty?
|
|
323
|
-
base = base.dup
|
|
324
|
-
base.path = '/'
|
|
325
|
-
end
|
|
326
|
-
self.class.new(base.join(relative_uri).normalize)
|
|
328
|
+
self.class.new(@uri.join(relative_uri).normalize)
|
|
327
329
|
end
|
|
328
330
|
|
|
329
331
|
protected
|
|
@@ -332,15 +334,13 @@ module Html2rss
|
|
|
332
334
|
#
|
|
333
335
|
# @return [String]
|
|
334
336
|
def document_identity
|
|
335
|
-
|
|
336
|
-
"#{uri.scheme}\0#{uri.authority}\0#{uri.path.to_s.chomp('/')}\0#{uri.query}"
|
|
337
|
+
"#{scheme}\0#{@uri.authority}\0#{path.chomp('/')}\0#{query}"
|
|
337
338
|
end
|
|
338
339
|
|
|
339
340
|
private
|
|
340
341
|
|
|
341
342
|
def ip_address_host?
|
|
342
|
-
IPAddr.new(host.
|
|
343
|
-
true
|
|
343
|
+
!IPAddr.new(host).nil?
|
|
344
344
|
rescue IPAddr::Error
|
|
345
345
|
false
|
|
346
346
|
end
|