html2rss 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/html2rss/article/deduplicator.rb +2 -4
- data/lib/html2rss/article.rb +16 -39
- data/lib/html2rss/auto_source/README.md +1 -1
- data/lib/html2rss/auto_source/cleanup.rb +15 -6
- data/lib/html2rss/auto_source/scraper/html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
- data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
- data/lib/html2rss/auto_source/scraper.rb +4 -14
- data/lib/html2rss/auto_source/segment.rb +7 -4
- data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
- data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
- data/lib/html2rss/auto_source/segmenter.rb +5 -5
- data/lib/html2rss/auto_source.rb +2 -2
- data/lib/html2rss/batch.rb +3 -1
- data/lib/html2rss/capture/README.md +6 -4
- data/lib/html2rss/capture.rb +67 -35
- data/lib/html2rss/channel.rb +3 -3
- data/lib/html2rss/cli/probe_view.rb +5 -3
- data/lib/html2rss/cli/render.rb +6 -5
- data/lib/html2rss/cli/validate.rb +28 -27
- data/lib/html2rss/cli.rb +1 -1
- data/lib/html2rss/config/issue_mapper.rb +108 -0
- data/lib/html2rss/config/request_headers.rb +2 -3
- data/lib/html2rss/config/schema.rb +16 -1
- data/lib/html2rss/config/selectors_validator.rb +92 -70
- data/lib/html2rss/config/validation_issue.rb +60 -0
- data/lib/html2rss/config/validation_report.rb +55 -0
- data/lib/html2rss/config/validator.rb +10 -6
- data/lib/html2rss/config.rb +35 -113
- data/lib/html2rss/defaults.rb +42 -30
- data/lib/html2rss/doctor/botasaurus.rb +6 -4
- data/lib/html2rss/doctor.rb +8 -0
- data/lib/html2rss/error.rb +1 -1
- data/lib/html2rss/feed_builder/rss.rb +9 -14
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
- data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
- data/lib/html2rss/feed_resolution/diag.rb +3 -1
- data/lib/html2rss/feed_resolution/options.rb +3 -1
- data/lib/html2rss/feed_resolution/policy.rb +2 -2
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +2 -4
- data/lib/html2rss/html/feed_link.rb +3 -1
- data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
- data/lib/html2rss/html/navigator.rb +10 -10
- data/lib/html2rss/html/rendering/description_builder.rb +1 -1
- data/lib/html2rss/html/sst_article_extractor.rb +8 -5
- data/lib/html2rss/link_destination/destination_facts.rb +4 -2
- data/lib/html2rss/link_destination/noise_policy.rb +10 -1
- data/lib/html2rss/link_destination/path_classifier.rb +45 -38
- data/lib/html2rss/mcp/README.md +2 -0
- data/lib/html2rss/mcp/config_argument.rb +3 -1
- data/lib/html2rss/mcp/contract.rb +6 -20
- data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
- data/lib/html2rss/mcp/outcome.rb +36 -43
- data/lib/html2rss/mcp/server/tools.rb +111 -131
- data/lib/html2rss/mcp/server.rb +26 -18
- data/lib/html2rss/page_recon/README.md +5 -5
- data/lib/html2rss/page_recon/diagnostics.rb +71 -81
- data/lib/html2rss/page_recon.rb +9 -5
- data/lib/html2rss/recon.rb +19 -30
- data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
- data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
- data/lib/html2rss/request_service/compressed_body.rb +4 -1
- data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
- data/lib/html2rss/request_service/network_guard.rb +7 -5
- data/lib/html2rss/request_service/response.rb +1 -2
- data/lib/html2rss/request_service/strategy.rb +3 -8
- data/lib/html2rss/scoring/container_assessor.rb +3 -1
- data/lib/html2rss/scoring/engine.rb +8 -8
- data/lib/html2rss/scoring/observation.rb +5 -3
- data/lib/html2rss/scoring/ranked_segment.rb +3 -1
- data/lib/html2rss/scrape_target.rb +3 -1
- data/lib/html2rss/selector_candidates.rb +234 -0
- data/lib/html2rss/selectors/README.md +64 -0
- data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
- data/lib/html2rss/selectors/extractors/href.rb +15 -16
- data/lib/html2rss/selectors/extractors/html.rb +8 -9
- data/lib/html2rss/selectors/extractors/static.rb +12 -14
- data/lib/html2rss/selectors/extractors/text.rb +8 -9
- data/lib/html2rss/selectors/extractors.rb +33 -20
- data/lib/html2rss/selectors/item_env.rb +31 -0
- data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
- data/lib/html2rss/selectors/option_spec.rb +140 -0
- data/lib/html2rss/selectors/post_processors/base.rb +79 -31
- data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
- data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
- data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
- data/lib/html2rss/selectors/post_processors/template.rb +23 -19
- data/lib/html2rss/selectors/post_processors.rb +6 -5
- data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
- data/lib/html2rss/selectors/step_env.rb +24 -0
- data/lib/html2rss/selectors.rb +231 -142
- data/lib/html2rss/sst/attrs.rb +5 -3
- data/lib/html2rss/sst/document.rb +3 -1
- data/lib/html2rss/sst/index.rb +3 -1
- data/lib/html2rss/sst/node.rb +29 -10
- data/lib/html2rss/sst/normalizer.rb +12 -16
- data/lib/html2rss/sst/tags.rb +7 -7
- data/lib/html2rss/status.rb +13 -8
- data/lib/html2rss/surface_category.rb +14 -32
- data/lib/html2rss/syndication/parser.rb +2 -2
- data/lib/html2rss/test/enhance_audit.rb +1 -3
- data/lib/html2rss/test/policy.rb +106 -0
- data/lib/html2rss/test.rb +126 -173
- data/lib/html2rss/url.rb +25 -4
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +21 -13
- data/lib/tasks/config_schema.rake +2 -2
- data/schema/html2rss-config.schema.json +11 -8
- metadata +15 -6
- data/lib/html2rss/selectors/item_scope.rb +0 -30
data/lib/html2rss/test.rb
CHANGED
|
@@ -12,27 +12,14 @@ module Html2rss
|
|
|
12
12
|
##
|
|
13
13
|
# Closed failure classification for a failed test run.
|
|
14
14
|
class FailureKind
|
|
15
|
-
#
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class << self
|
|
19
|
-
##
|
|
20
|
-
# @param value [FailureKind, Symbol, String]
|
|
21
|
-
# @return [FailureKind]
|
|
22
|
-
def coerce(value)
|
|
23
|
-
return value if value.is_a?(self)
|
|
24
|
-
|
|
25
|
-
new(name: value.to_sym)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
15
|
+
# Frozen name → instance table.
|
|
16
|
+
ALL = %i[schema execution min_items quality].to_h { |name| [name, new(name:)] }.freeze
|
|
17
|
+
private_class_method :new
|
|
28
18
|
|
|
29
19
|
##
|
|
30
|
-
# @param name [Symbol]
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
super
|
|
35
|
-
end
|
|
20
|
+
# @param name [Symbol, String]
|
|
21
|
+
# @return [FailureKind]
|
|
22
|
+
def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown failure kind: #{name.inspect}" }
|
|
36
23
|
|
|
37
24
|
##
|
|
38
25
|
# @return [Boolean]
|
|
@@ -59,19 +46,11 @@ module Html2rss
|
|
|
59
46
|
def to_s = name.to_s
|
|
60
47
|
end
|
|
61
48
|
|
|
49
|
+
QualityReport = Data.define(:warnings, :metrics, :native_feed, :defer_reason)
|
|
50
|
+
|
|
62
51
|
##
|
|
63
52
|
# Ship-quality audit summary for a configuration test (warn-only).
|
|
64
|
-
QualityReport
|
|
65
|
-
##
|
|
66
|
-
# @return [Hash{Symbol => Object}]
|
|
67
|
-
def to_h
|
|
68
|
-
{
|
|
69
|
-
warnings: warnings.map(&:to_s),
|
|
70
|
-
metrics:,
|
|
71
|
-
**(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
|
|
72
|
-
}.compact
|
|
73
|
-
end
|
|
74
|
-
|
|
53
|
+
class QualityReport
|
|
75
54
|
##
|
|
76
55
|
# @param audit [Html2rss::AutoSource::Cleanup::AuditResult]
|
|
77
56
|
# @param native_feed [String, nil]
|
|
@@ -85,11 +64,18 @@ module Html2rss
|
|
|
85
64
|
end
|
|
86
65
|
new(warnings: report_warnings.freeze, metrics: audit.metrics, native_feed:, defer_reason:)
|
|
87
66
|
end
|
|
67
|
+
|
|
68
|
+
##
|
|
69
|
+
# @return [Hash{Symbol => Object}]
|
|
70
|
+
def to_h
|
|
71
|
+
{
|
|
72
|
+
warnings: warnings.map(&:to_s),
|
|
73
|
+
metrics:,
|
|
74
|
+
**(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
|
|
75
|
+
}.compact
|
|
76
|
+
end
|
|
88
77
|
end
|
|
89
78
|
|
|
90
|
-
##
|
|
91
|
-
# Immutable outcome of a configuration test. Success carries +rss+ XML from the
|
|
92
|
-
# first live extraction; failures carry a typed {FailureKind}.
|
|
93
79
|
Result = Data.define(
|
|
94
80
|
:success,
|
|
95
81
|
:item_count,
|
|
@@ -98,13 +84,20 @@ module Html2rss
|
|
|
98
84
|
:channel_url,
|
|
99
85
|
:strategy_used,
|
|
100
86
|
:duration_seconds,
|
|
101
|
-
:
|
|
87
|
+
:validation_issues,
|
|
102
88
|
:error_message,
|
|
103
89
|
:failure_kind,
|
|
104
90
|
:rss,
|
|
105
91
|
:quality_report,
|
|
106
|
-
:enhance_compare
|
|
107
|
-
|
|
92
|
+
:enhance_compare,
|
|
93
|
+
:response_body
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
##
|
|
97
|
+
# Immutable outcome of a configuration test. Success carries +rss+ XML from the
|
|
98
|
+
# first live extraction; failures carry a typed {FailureKind}.
|
|
99
|
+
# +response_body+ is a copy of the fetched page for in-process readers and stays off {#to_h}.
|
|
100
|
+
class Result
|
|
108
101
|
##
|
|
109
102
|
# @param success [Boolean]
|
|
110
103
|
# @param item_count [Integer]
|
|
@@ -113,22 +106,23 @@ module Html2rss
|
|
|
113
106
|
# @param channel_url [String, nil]
|
|
114
107
|
# @param strategy_used [Symbol, nil]
|
|
115
108
|
# @param duration_seconds [Float]
|
|
116
|
-
# @param
|
|
109
|
+
# @param validation_issues [Array<Html2rss::Config::ValidationIssue>, nil]
|
|
117
110
|
# @param error_message [String, nil]
|
|
118
111
|
# @param failure_kind [FailureKind, nil]
|
|
119
112
|
# @param rss [String, nil]
|
|
120
113
|
# @param quality_report [QualityReport, nil]
|
|
121
114
|
# @param enhance_compare [Hash, nil]
|
|
115
|
+
# @param response_body [String, nil] fetched page HTML copied from the pipeline response
|
|
122
116
|
def initialize(success:, item_count:, sample_items:, channel_title:, channel_url:, # rubocop:disable Metrics/ParameterLists
|
|
123
|
-
strategy_used:, duration_seconds:,
|
|
124
|
-
failure_kind:, rss:, quality_report: nil, enhance_compare: nil)
|
|
117
|
+
strategy_used:, duration_seconds:, validation_issues:, error_message:,
|
|
118
|
+
failure_kind:, rss:, quality_report: nil, enhance_compare: nil, response_body: nil)
|
|
125
119
|
super
|
|
126
120
|
end
|
|
127
121
|
|
|
128
122
|
##
|
|
129
123
|
# @return [Boolean] whether the schema validation succeeded
|
|
130
124
|
def valid_schema?
|
|
131
|
-
|
|
125
|
+
validation_issues.nil? || validation_issues.empty?
|
|
132
126
|
end
|
|
133
127
|
|
|
134
128
|
##
|
|
@@ -138,8 +132,8 @@ module Html2rss
|
|
|
138
132
|
end
|
|
139
133
|
|
|
140
134
|
##
|
|
141
|
-
# @return [Hash{Symbol => Object}] hash
|
|
142
|
-
def to_h # rubocop:disable Metrics/MethodLength
|
|
135
|
+
# @return [Hash{Symbol => Object}] public test hash; +response_body+ stays off this wire
|
|
136
|
+
def to_h # rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- wire serialization omits response_body
|
|
143
137
|
{
|
|
144
138
|
success:,
|
|
145
139
|
item_count:,
|
|
@@ -148,7 +142,7 @@ module Html2rss
|
|
|
148
142
|
channel_url:,
|
|
149
143
|
strategy_used:,
|
|
150
144
|
duration_seconds:,
|
|
151
|
-
|
|
145
|
+
validation_issues: validation_issues&.map(&:to_h),
|
|
152
146
|
error_message:,
|
|
153
147
|
failure_kind: failure_kind&.to_sym,
|
|
154
148
|
rss:,
|
|
@@ -169,93 +163,113 @@ module Html2rss
|
|
|
169
163
|
# @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
|
|
170
164
|
# @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
|
|
171
165
|
# @return [Html2rss::Test::Result]
|
|
172
|
-
def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
166
|
+
def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
173
167
|
strict_quality: false, compare_enhance: false)
|
|
174
168
|
raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
|
|
175
|
-
return validation_failure_result(validation
|
|
169
|
+
return validation_failure_result(validation, raw_config) unless validation.success?
|
|
176
170
|
|
|
171
|
+
config = build_test_config(raw_config, strategy, params)
|
|
172
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
173
|
+
execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
|
|
174
|
+
rescue StandardError => error
|
|
175
|
+
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
176
|
+
execution_failure_result(error, raw_config, duration)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def build_test_config(raw_config, strategy, params)
|
|
177
180
|
raw_config[:strategy] = strategy.to_sym if strategy
|
|
178
181
|
raw_config[:params] = params if params&.any?
|
|
182
|
+
Config.from_hash(raw_config)
|
|
183
|
+
end
|
|
184
|
+
private_class_method :build_test_config
|
|
185
|
+
|
|
186
|
+
def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
187
|
+
quality_report_for(
|
|
188
|
+
feed_result.to_rss.items,
|
|
189
|
+
channel_url: raw_config.dig(:channel, :url).to_s,
|
|
190
|
+
config:,
|
|
191
|
+
feed_result:,
|
|
192
|
+
pipeline_outcome:
|
|
193
|
+
)
|
|
194
|
+
end
|
|
195
|
+
private_class_method :compile_quality_report
|
|
179
196
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
channel_url:,
|
|
216
|
-
strategy_used:,
|
|
217
|
-
duration_seconds: duration.round(3),
|
|
218
|
-
validation_errors: nil,
|
|
219
|
-
error_message:,
|
|
220
|
-
failure_kind:,
|
|
221
|
-
rss: passed ? rss_xml : nil,
|
|
222
|
-
quality_report:,
|
|
223
|
-
enhance_compare:
|
|
224
|
-
)
|
|
225
|
-
rescue StandardError => error
|
|
226
|
-
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
227
|
-
execution_failure_result(error, raw_config, duration)
|
|
228
|
-
end
|
|
197
|
+
def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
|
|
198
|
+
compare_enhance:)
|
|
199
|
+
feed_result, pipeline_outcome = extract_feed(raw_config)
|
|
200
|
+
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
201
|
+
quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
202
|
+
enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
|
|
203
|
+
|
|
204
|
+
build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
|
|
205
|
+
min_items:, strict_quality:, response_body: pipeline_outcome&.response&.body)
|
|
206
|
+
end
|
|
207
|
+
private_class_method :execute_timed_pipeline
|
|
208
|
+
|
|
209
|
+
def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
210
|
+
min_items:, strict_quality:, response_body:)
|
|
211
|
+
rss_doc = feed_result.to_rss
|
|
212
|
+
outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
|
|
213
|
+
strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
|
|
214
|
+
RequestService.default_strategy_name
|
|
215
|
+
|
|
216
|
+
Result.new(
|
|
217
|
+
success: outcome.passed,
|
|
218
|
+
item_count: rss_doc.items.size,
|
|
219
|
+
sample_items: extract_samples(rss_doc.items),
|
|
220
|
+
channel_title: feed_result.channel_title,
|
|
221
|
+
channel_url: raw_config.dig(:channel, :url).to_s,
|
|
222
|
+
strategy_used:,
|
|
223
|
+
duration_seconds: duration.round(3),
|
|
224
|
+
validation_issues: nil,
|
|
225
|
+
error_message: outcome.error_message,
|
|
226
|
+
failure_kind: outcome.failure_kind,
|
|
227
|
+
rss: outcome.passed ? rss_doc.to_s : nil,
|
|
228
|
+
quality_report:,
|
|
229
|
+
enhance_compare:,
|
|
230
|
+
response_body:
|
|
231
|
+
)
|
|
229
232
|
end
|
|
233
|
+
private_class_method :build_test_result
|
|
230
234
|
|
|
231
235
|
def extract_samples(items, limit: 3)
|
|
232
236
|
items.first(limit).map do |item|
|
|
237
|
+
article = feed_item_as_article(item)
|
|
233
238
|
{
|
|
234
|
-
title:
|
|
235
|
-
url:
|
|
236
|
-
published_at:
|
|
239
|
+
title: article.title.to_s.strip,
|
|
240
|
+
url: article.url.to_s,
|
|
241
|
+
published_at: article.published_at
|
|
237
242
|
}
|
|
238
243
|
end
|
|
239
244
|
end
|
|
240
245
|
private_class_method :extract_samples
|
|
241
246
|
|
|
247
|
+
def feed_item_as_article(item)
|
|
248
|
+
case item
|
|
249
|
+
when Article then item
|
|
250
|
+
else
|
|
251
|
+
Article.new(title: item.title, url: item.link, published_at: item.pubDate)
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
private_class_method :feed_item_as_article
|
|
255
|
+
|
|
242
256
|
##
|
|
243
257
|
# Builds a ship-quality report for RSS items (shared by test and MCP apply).
|
|
244
258
|
#
|
|
245
259
|
# @param items [Array]
|
|
246
260
|
# @param channel_url [String]
|
|
247
|
-
# @param
|
|
261
|
+
# @param config [Html2rss::Config]
|
|
248
262
|
# @param feed_result [Html2rss::FeedResult]
|
|
249
263
|
# @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
|
|
250
264
|
# @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
|
|
251
265
|
# @return [QualityReport]
|
|
252
|
-
def quality_report_for(items, channel_url:,
|
|
266
|
+
def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
|
|
253
267
|
probe_native_feed: true)
|
|
254
268
|
audit = AutoSource::Cleanup.audit_feed_items(items)
|
|
255
|
-
native_feed = probe_native_feed ? probe_native_feed_url(channel_url,
|
|
269
|
+
native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
|
|
256
270
|
report = QualityReport.from_audit(audit, native_feed:)
|
|
257
271
|
report = append_url_mismatch_warning(report, channel_url, feed_result)
|
|
258
|
-
merge_enhance_audit(report,
|
|
272
|
+
merge_enhance_audit(report, config, pipeline_outcome)
|
|
259
273
|
end
|
|
260
274
|
|
|
261
275
|
def extract_feed(raw_config)
|
|
@@ -264,18 +278,15 @@ module Html2rss
|
|
|
264
278
|
end
|
|
265
279
|
private_class_method :extract_feed
|
|
266
280
|
|
|
267
|
-
def enhance_enabled?(
|
|
268
|
-
config = Config.from_hash(raw_config)
|
|
281
|
+
def enhance_enabled?(config)
|
|
269
282
|
return false unless config.selectors
|
|
270
283
|
|
|
271
284
|
!!config.selectors.dig(:items, :enhance)
|
|
272
285
|
end
|
|
273
286
|
private_class_method :enhance_enabled?
|
|
274
287
|
|
|
275
|
-
def build_enhance_compare(
|
|
288
|
+
def build_enhance_compare(config, pipeline_outcome)
|
|
276
289
|
return nil unless pipeline_outcome
|
|
277
|
-
|
|
278
|
-
config = Config.from_hash(raw_config)
|
|
279
290
|
return nil unless config.selectors
|
|
280
291
|
|
|
281
292
|
EnhanceAudit.compare(
|
|
@@ -286,10 +297,8 @@ module Html2rss
|
|
|
286
297
|
end
|
|
287
298
|
private_class_method :build_enhance_compare
|
|
288
299
|
|
|
289
|
-
def merge_enhance_audit(report,
|
|
290
|
-
return report unless pipeline_outcome && enhance_enabled?(
|
|
291
|
-
|
|
292
|
-
config = Config.from_hash(raw_config)
|
|
300
|
+
def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
|
|
301
|
+
return report unless pipeline_outcome && enhance_enabled?(config)
|
|
293
302
|
return report unless config.selectors
|
|
294
303
|
|
|
295
304
|
slice = EnhanceAudit.probe(
|
|
@@ -350,67 +359,11 @@ module Html2rss
|
|
|
350
359
|
end
|
|
351
360
|
private_class_method :url_mismatch?
|
|
352
361
|
|
|
353
|
-
def
|
|
354
|
-
metrics = quality_report.metrics
|
|
355
|
-
item_count = metrics[:item_count]
|
|
356
|
-
return false if item_count.zero?
|
|
357
|
-
|
|
358
|
-
duplicate_urls_failure?(metrics) || junk_title_ratio_failure?(metrics) || short_title_failure?(metrics)
|
|
359
|
-
end
|
|
360
|
-
private_class_method :quality_failure?
|
|
361
|
-
|
|
362
|
-
def duplicate_urls_failure?(metrics)
|
|
363
|
-
metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
|
|
364
|
-
end
|
|
365
|
-
private_class_method :duplicate_urls_failure?
|
|
366
|
-
|
|
367
|
-
def junk_title_ratio_failure?(metrics)
|
|
368
|
-
metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
|
|
369
|
-
end
|
|
370
|
-
private_class_method :junk_title_ratio_failure?
|
|
371
|
-
|
|
372
|
-
def short_title_failure?(metrics)
|
|
373
|
-
metrics[:short_title_count].positive?
|
|
374
|
-
end
|
|
375
|
-
private_class_method :short_title_failure?
|
|
376
|
-
|
|
377
|
-
def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
|
|
378
|
-
return [nil, nil] if min_items_passed && !quality_failed
|
|
379
|
-
unless min_items_passed
|
|
380
|
-
return [FailureKind.coerce(:min_items),
|
|
381
|
-
"Extracted #{item_count} items (minimum required: #{min_items})"]
|
|
382
|
-
end
|
|
383
|
-
|
|
384
|
-
[FailureKind.coerce(:quality), quality_failure_message(quality_report)]
|
|
385
|
-
end
|
|
386
|
-
private_class_method :outcome_failure
|
|
387
|
-
|
|
388
|
-
def quality_failure_message(quality_report)
|
|
389
|
-
reasons = quality_failure_reasons(quality_report.metrics)
|
|
390
|
-
"Feed quality check failed (#{reasons.join(', ')})"
|
|
391
|
-
end
|
|
392
|
-
private_class_method :quality_failure_message
|
|
393
|
-
|
|
394
|
-
def quality_failure_reasons(metrics)
|
|
395
|
-
reasons = []
|
|
396
|
-
reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
|
|
397
|
-
reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
|
|
398
|
-
reasons << 'short_titles' if short_title_failure?(metrics)
|
|
399
|
-
reasons
|
|
400
|
-
end
|
|
401
|
-
private_class_method :quality_failure_reasons
|
|
402
|
-
|
|
403
|
-
def log_strict_quality_failure(failure_kind, item_count)
|
|
404
|
-
Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
|
|
405
|
-
end
|
|
406
|
-
private_class_method :log_strict_quality_failure
|
|
407
|
-
|
|
408
|
-
def probe_native_feed_url(channel_url, raw_config)
|
|
362
|
+
def probe_native_feed_url(channel_url, config)
|
|
409
363
|
return nil if channel_url.to_s.empty?
|
|
410
364
|
|
|
411
|
-
config = Config.from_hash(raw_config)
|
|
412
365
|
resources = FeedPipeline::RuntimePolicy.resources_for(config)
|
|
413
|
-
strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(
|
|
366
|
+
strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(config.strategy)
|
|
414
367
|
session = RequestSession.build(
|
|
415
368
|
config:, strategy:, budget: resources.budget, policy: resources.policy
|
|
416
369
|
)
|
|
@@ -420,7 +373,7 @@ module Html2rss
|
|
|
420
373
|
end
|
|
421
374
|
private_class_method :probe_native_feed_url
|
|
422
375
|
|
|
423
|
-
def validation_failure_result(
|
|
376
|
+
def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
|
|
424
377
|
Result.new(
|
|
425
378
|
success: false,
|
|
426
379
|
item_count: 0,
|
|
@@ -429,9 +382,9 @@ module Html2rss
|
|
|
429
382
|
channel_url: raw_config.dig(:channel, :url),
|
|
430
383
|
strategy_used: raw_config[:strategy],
|
|
431
384
|
duration_seconds: 0.0,
|
|
432
|
-
|
|
385
|
+
validation_issues: report.issues,
|
|
433
386
|
error_message: 'Configuration schema validation failed',
|
|
434
|
-
failure_kind: FailureKind
|
|
387
|
+
failure_kind: FailureKind[:schema],
|
|
435
388
|
rss: nil,
|
|
436
389
|
quality_report: nil
|
|
437
390
|
)
|
|
@@ -447,9 +400,9 @@ module Html2rss
|
|
|
447
400
|
channel_url: raw_config.dig(:channel, :url),
|
|
448
401
|
strategy_used: raw_config[:strategy],
|
|
449
402
|
duration_seconds: duration.round(3),
|
|
450
|
-
|
|
403
|
+
validation_issues: nil,
|
|
451
404
|
error_message: "#{error.class}: #{error.message}",
|
|
452
|
-
failure_kind: FailureKind
|
|
405
|
+
failure_kind: FailureKind[:execution],
|
|
453
406
|
rss: nil,
|
|
454
407
|
quality_report: nil
|
|
455
408
|
)
|
data/lib/html2rss/url.rb
CHANGED
|
@@ -42,14 +42,21 @@ module Html2rss
|
|
|
42
42
|
url = Addressable::URI.parse(relative_url.to_s.strip)
|
|
43
43
|
return new(url) if url.absolute?
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
base_uri.path = '/' if base_uri.path.empty?
|
|
47
|
-
|
|
48
|
-
new(base_uri.join(url).normalize)
|
|
45
|
+
base_url.is_a?(self) ? base_url.join_uri(url) : join_parsed_base(base_url, url)
|
|
49
46
|
rescue Addressable::URI::InvalidURIError
|
|
50
47
|
raise ArgumentError, 'URL could not be parsed'
|
|
51
48
|
end
|
|
52
49
|
|
|
50
|
+
# @param base_url [String]
|
|
51
|
+
# @param url [Addressable::URI]
|
|
52
|
+
# @return [Url]
|
|
53
|
+
def self.join_parsed_base(base_url, url)
|
|
54
|
+
uri = Addressable::URI.parse(base_url.to_s)
|
|
55
|
+
uri.path = '/' if uri.path.empty?
|
|
56
|
+
new(uri.join(url).normalize)
|
|
57
|
+
end
|
|
58
|
+
private_class_method :join_parsed_base
|
|
59
|
+
|
|
53
60
|
##
|
|
54
61
|
# Creates a URL by sanitizing a raw URL string.
|
|
55
62
|
# Removes spaces and extracts the first valid URL from the string.
|
|
@@ -305,6 +312,20 @@ module Html2rss
|
|
|
305
312
|
# @return [String] the debug representation
|
|
306
313
|
def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
|
|
307
314
|
|
|
315
|
+
##
|
|
316
|
+
# Resolves +relative_uri+ against this URL's already-parsed Addressable URI.
|
|
317
|
+
#
|
|
318
|
+
# @param relative_uri [Addressable::URI]
|
|
319
|
+
# @return [Url]
|
|
320
|
+
def join_uri(relative_uri)
|
|
321
|
+
base = @uri
|
|
322
|
+
if base.path.empty?
|
|
323
|
+
base = base.dup
|
|
324
|
+
base.path = '/'
|
|
325
|
+
end
|
|
326
|
+
self.class.new(base.join(relative_uri).normalize)
|
|
327
|
+
end
|
|
328
|
+
|
|
308
329
|
protected
|
|
309
330
|
|
|
310
331
|
# Scheme, authority, slash-stripped path, and query (fragment omitted).
|
data/lib/html2rss/version.rb
CHANGED
data/lib/html2rss.rb
CHANGED
|
@@ -11,20 +11,30 @@ loader.inflector.inflect(
|
|
|
11
11
|
loader.setup
|
|
12
12
|
|
|
13
13
|
require 'logger'
|
|
14
|
-
require 'forwardable'
|
|
15
14
|
require 'html2rss/defaults'
|
|
16
15
|
|
|
17
16
|
##
|
|
18
17
|
# The Html2rss namespace.
|
|
19
18
|
module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
20
19
|
##
|
|
21
|
-
#
|
|
20
|
+
# Forwards debug/info/warn/error to {Html2rss.logger}.
|
|
22
21
|
module Log
|
|
23
22
|
class << self
|
|
24
|
-
|
|
23
|
+
##
|
|
24
|
+
# @return [true]
|
|
25
|
+
def debug(...) = Html2rss.logger.debug(...)
|
|
25
26
|
|
|
26
|
-
|
|
27
|
-
|
|
27
|
+
##
|
|
28
|
+
# @return [true]
|
|
29
|
+
def info(...) = Html2rss.logger.info(...)
|
|
30
|
+
|
|
31
|
+
##
|
|
32
|
+
# @return [true]
|
|
33
|
+
def warn(...) = Html2rss.logger.warn(...)
|
|
34
|
+
|
|
35
|
+
##
|
|
36
|
+
# @return [true]
|
|
37
|
+
def error(...) = Html2rss.logger.error(...)
|
|
28
38
|
end
|
|
29
39
|
end
|
|
30
40
|
|
|
@@ -79,7 +89,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
79
89
|
# @param config_input [Hash, String]
|
|
80
90
|
# @param feed_name [String, nil]
|
|
81
91
|
# @param params [Hash]
|
|
82
|
-
# @return [
|
|
92
|
+
# @return [Html2rss::Config::ValidationReport]
|
|
83
93
|
def self.validate(config_input, feed_name = nil, params: {})
|
|
84
94
|
_raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
|
|
85
95
|
validation
|
|
@@ -179,7 +189,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
179
189
|
# @param pretty [Boolean] whether to pretty-print JSON
|
|
180
190
|
# @return [String]
|
|
181
191
|
def self.schema_json(pretty: true)
|
|
182
|
-
Config.json_schema_json(pretty:)
|
|
192
|
+
Config::Schema.json_schema_json(pretty:)
|
|
183
193
|
end
|
|
184
194
|
|
|
185
195
|
##
|
|
@@ -319,26 +329,24 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
319
329
|
end
|
|
320
330
|
|
|
321
331
|
##
|
|
322
|
-
# @return [
|
|
332
|
+
# @return [Logger] the logger
|
|
323
333
|
def logger
|
|
324
334
|
defaults.logger
|
|
325
335
|
end
|
|
326
336
|
|
|
327
337
|
##
|
|
328
|
-
# @param logger [
|
|
338
|
+
# @param logger [Logger] the new logger
|
|
329
339
|
def logger=(logger)
|
|
330
340
|
configure { |config| config.logger = logger }
|
|
331
341
|
end
|
|
332
342
|
|
|
333
|
-
private
|
|
334
|
-
|
|
335
343
|
##
|
|
336
344
|
# Resets the global defaults (mainly for testing).
|
|
337
345
|
#
|
|
338
346
|
# @return [void]
|
|
339
347
|
def reset_defaults!
|
|
340
348
|
@defaults = nil
|
|
341
|
-
logger.level = defaults.log_level
|
|
349
|
+
logger.level = defaults.log_level
|
|
342
350
|
end
|
|
343
351
|
end
|
|
344
352
|
|
|
@@ -362,7 +370,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
362
370
|
end
|
|
363
371
|
end
|
|
364
372
|
|
|
365
|
-
logger.level = defaults.log_level
|
|
373
|
+
logger.level = defaults.log_level
|
|
366
374
|
end
|
|
367
375
|
|
|
368
376
|
loader.eager_load
|
|
@@ -7,10 +7,10 @@ require_relative '../html2rss'
|
|
|
7
7
|
namespace :config do
|
|
8
8
|
desc 'Generate config JSON schema'
|
|
9
9
|
task :schema do
|
|
10
|
-
destination = Html2rss::Config.
|
|
10
|
+
destination = Html2rss::Config::Schema.path
|
|
11
11
|
|
|
12
12
|
FileUtils.mkdir_p(File.dirname(destination))
|
|
13
|
-
File.write(destination, "#{Html2rss::Config.json_schema_json}\n")
|
|
13
|
+
File.write(destination, "#{Html2rss::Config::Schema.json_schema_json}\n")
|
|
14
14
|
|
|
15
15
|
puts "Generated config schema at #{destination}"
|
|
16
16
|
end
|
|
@@ -434,7 +434,8 @@
|
|
|
434
434
|
"properties": {
|
|
435
435
|
"selector": {
|
|
436
436
|
"type": "string",
|
|
437
|
-
"minLength": 1
|
|
437
|
+
"minLength": 1,
|
|
438
|
+
"default": "a[href]"
|
|
438
439
|
},
|
|
439
440
|
"order": {
|
|
440
441
|
"enum": [
|
|
@@ -507,9 +508,7 @@
|
|
|
507
508
|
]
|
|
508
509
|
}
|
|
509
510
|
},
|
|
510
|
-
"required": [
|
|
511
|
-
"selector"
|
|
512
|
-
],
|
|
511
|
+
"required": [],
|
|
513
512
|
"description": "Defines the items selector and list-card enhance settings."
|
|
514
513
|
},
|
|
515
514
|
"enclosure": {
|
|
@@ -848,7 +847,7 @@
|
|
|
848
847
|
"gsub": {
|
|
849
848
|
"type": "object",
|
|
850
849
|
"title": "gsub",
|
|
851
|
-
"description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string).",
|
|
850
|
+
"description": "Replace matches of `pattern` in the extracted string with `replacement` (Ruby String#gsub; pattern may be a regexp-like string). Patterns over 256 characters as written, or with nested quantifiers, are rejected.",
|
|
852
851
|
"examples": [
|
|
853
852
|
{
|
|
854
853
|
"name": "gsub",
|
|
@@ -862,10 +861,14 @@
|
|
|
862
861
|
"const": "gsub"
|
|
863
862
|
},
|
|
864
863
|
"pattern": {
|
|
865
|
-
"type": "string"
|
|
864
|
+
"type": "string",
|
|
865
|
+
"maxLength": 256
|
|
866
866
|
},
|
|
867
867
|
"replacement": {
|
|
868
|
-
"type":
|
|
868
|
+
"type": [
|
|
869
|
+
"string",
|
|
870
|
+
"object"
|
|
871
|
+
]
|
|
869
872
|
}
|
|
870
873
|
},
|
|
871
874
|
"required": [
|
|
@@ -1024,7 +1027,7 @@
|
|
|
1024
1027
|
"type": "string",
|
|
1025
1028
|
"const": "href",
|
|
1026
1029
|
"title": "href",
|
|
1027
|
-
"description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the
|
|
1030
|
+
"description": "Return the absolute URL from the selected element's `href` attribute (relative hrefs are resolved against the page base URL).",
|
|
1028
1031
|
"examples": [
|
|
1029
1032
|
"href"
|
|
1030
1033
|
]
|