html2rss 0.29.1 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -8
- data/html2rss.gemspec +1 -3
- data/lib/html2rss/article/deduplicator.rb +2 -4
- data/lib/html2rss/article.rb +16 -39
- data/lib/html2rss/auto_source/README.md +1 -1
- data/lib/html2rss/auto_source/cleanup.rb +15 -6
- data/lib/html2rss/auto_source/scraper/html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
- data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
- data/lib/html2rss/auto_source/scraper.rb +4 -14
- data/lib/html2rss/auto_source/segment.rb +7 -4
- data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
- data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
- data/lib/html2rss/auto_source/segmenter.rb +5 -5
- data/lib/html2rss/auto_source.rb +2 -2
- data/lib/html2rss/batch.rb +6 -4
- data/lib/html2rss/capture/README.md +7 -5
- data/lib/html2rss/capture.rb +68 -36
- data/lib/html2rss/channel.rb +3 -3
- data/lib/html2rss/cli/probe_view.rb +5 -3
- data/lib/html2rss/cli/render.rb +6 -5
- data/lib/html2rss/cli/validate.rb +28 -27
- data/lib/html2rss/cli.rb +1 -1
- data/lib/html2rss/config/issue_mapper.rb +108 -0
- data/lib/html2rss/config/request_headers.rb +6 -4
- data/lib/html2rss/config/schema.rb +16 -1
- data/lib/html2rss/config/selectors_validator.rb +92 -70
- data/lib/html2rss/config/validation_issue.rb +60 -0
- data/lib/html2rss/config/validation_report.rb +55 -0
- data/lib/html2rss/config/validator.rb +10 -6
- data/lib/html2rss/config.rb +35 -101
- data/lib/html2rss/defaults.rb +42 -30
- data/lib/html2rss/doctor/botasaurus.rb +19 -9
- data/lib/html2rss/doctor.rb +8 -0
- data/lib/html2rss/error.rb +1 -1
- data/lib/html2rss/feed_builder/rss.rb +9 -14
- data/lib/html2rss/feed_pipeline/README.md +8 -8
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +1 -1
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +7 -3
- data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
- data/lib/html2rss/feed_resolution/diag.rb +3 -1
- data/lib/html2rss/feed_resolution/options.rb +3 -1
- data/lib/html2rss/feed_resolution/policy.rb +2 -2
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +2 -4
- data/lib/html2rss/html/feed_link.rb +3 -1
- data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
- data/lib/html2rss/html/navigator.rb +10 -10
- data/lib/html2rss/html/rendering/description_builder.rb +1 -1
- data/lib/html2rss/html/sst_article_extractor.rb +8 -5
- data/lib/html2rss/link_destination/destination_facts.rb +4 -2
- data/lib/html2rss/link_destination/noise_policy.rb +10 -1
- data/lib/html2rss/link_destination/path_classifier.rb +45 -38
- data/lib/html2rss/mcp/README.md +5 -3
- data/lib/html2rss/mcp/config_argument.rb +3 -1
- data/lib/html2rss/mcp/contract.rb +15 -25
- data/lib/html2rss/mcp/outcome/playbook.rb +7 -7
- data/lib/html2rss/mcp/outcome.rb +36 -43
- data/lib/html2rss/mcp/server/tools.rb +112 -132
- data/lib/html2rss/mcp/server.rb +27 -19
- data/lib/html2rss/page_recon/README.md +5 -5
- data/lib/html2rss/page_recon/diagnostics.rb +71 -81
- data/lib/html2rss/page_recon.rb +9 -5
- data/lib/html2rss/recon.rb +20 -31
- data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
- data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
- data/lib/html2rss/request_service/compressed_body.rb +17 -9
- data/lib/html2rss/request_service/httpx_strategy.rb +239 -0
- data/lib/html2rss/request_service/network_guard.rb +7 -5
- data/lib/html2rss/request_service/response.rb +1 -2
- data/lib/html2rss/request_service/response_guard.rb +0 -16
- data/lib/html2rss/request_service/strategy.rb +22 -11
- data/lib/html2rss/request_service.rb +46 -85
- data/lib/html2rss/scoring/container_assessor.rb +3 -1
- data/lib/html2rss/scoring/engine.rb +8 -8
- data/lib/html2rss/scoring/observation.rb +5 -3
- data/lib/html2rss/scoring/ranked_segment.rb +3 -1
- data/lib/html2rss/scrape_target.rb +3 -1
- data/lib/html2rss/selector_candidates.rb +234 -0
- data/lib/html2rss/selectors/README.md +64 -0
- data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
- data/lib/html2rss/selectors/extractors/href.rb +15 -16
- data/lib/html2rss/selectors/extractors/html.rb +8 -9
- data/lib/html2rss/selectors/extractors/static.rb +12 -14
- data/lib/html2rss/selectors/extractors/text.rb +8 -9
- data/lib/html2rss/selectors/extractors.rb +33 -20
- data/lib/html2rss/selectors/item_env.rb +31 -0
- data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
- data/lib/html2rss/selectors/option_spec.rb +140 -0
- data/lib/html2rss/selectors/post_processors/base.rb +79 -31
- data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
- data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
- data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
- data/lib/html2rss/selectors/post_processors/template.rb +23 -19
- data/lib/html2rss/selectors/post_processors.rb +6 -5
- data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
- data/lib/html2rss/selectors/step_env.rb +24 -0
- data/lib/html2rss/selectors.rb +231 -142
- data/lib/html2rss/sst/attrs.rb +5 -3
- data/lib/html2rss/sst/document.rb +3 -1
- data/lib/html2rss/sst/index.rb +3 -1
- data/lib/html2rss/sst/node.rb +29 -10
- data/lib/html2rss/sst/normalizer.rb +12 -16
- data/lib/html2rss/sst/tags.rb +7 -7
- data/lib/html2rss/status.rb +13 -8
- data/lib/html2rss/surface_category.rb +14 -32
- data/lib/html2rss/syndication/parser.rb +2 -2
- data/lib/html2rss/test/enhance_audit.rb +1 -3
- data/lib/html2rss/test/policy.rb +106 -0
- data/lib/html2rss/test.rb +126 -172
- data/lib/html2rss/url.rb +25 -4
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +27 -19
- data/lib/tasks/config_schema.rake +2 -2
- data/schema/html2rss-config.schema.json +11 -8
- metadata +19 -44
- data/lib/html2rss/request_service/faraday_strategy.rb +0 -233
- data/lib/html2rss/selectors/item_scope.rb +0 -30
data/lib/html2rss/test.rb
CHANGED
|
@@ -12,27 +12,14 @@ module Html2rss
|
|
|
12
12
|
##
|
|
13
13
|
# Closed failure classification for a failed test run.
|
|
14
14
|
class FailureKind
|
|
15
|
-
#
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class << self
|
|
19
|
-
##
|
|
20
|
-
# @param value [FailureKind, Symbol, String]
|
|
21
|
-
# @return [FailureKind]
|
|
22
|
-
def coerce(value)
|
|
23
|
-
return value if value.is_a?(self)
|
|
24
|
-
|
|
25
|
-
new(name: value.to_sym)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
15
|
+
# Frozen name → instance table.
|
|
16
|
+
ALL = %i[schema execution min_items quality].to_h { |name| [name, new(name:)] }.freeze
|
|
17
|
+
private_class_method :new
|
|
28
18
|
|
|
29
19
|
##
|
|
30
|
-
# @param name [Symbol]
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
super
|
|
35
|
-
end
|
|
20
|
+
# @param name [Symbol, String]
|
|
21
|
+
# @return [FailureKind]
|
|
22
|
+
def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown failure kind: #{name.inspect}" }
|
|
36
23
|
|
|
37
24
|
##
|
|
38
25
|
# @return [Boolean]
|
|
@@ -59,19 +46,11 @@ module Html2rss
|
|
|
59
46
|
def to_s = name.to_s
|
|
60
47
|
end
|
|
61
48
|
|
|
49
|
+
QualityReport = Data.define(:warnings, :metrics, :native_feed, :defer_reason)
|
|
50
|
+
|
|
62
51
|
##
|
|
63
52
|
# Ship-quality audit summary for a configuration test (warn-only).
|
|
64
|
-
QualityReport
|
|
65
|
-
##
|
|
66
|
-
# @return [Hash{Symbol => Object}]
|
|
67
|
-
def to_h
|
|
68
|
-
{
|
|
69
|
-
warnings: warnings.map(&:to_s),
|
|
70
|
-
metrics:,
|
|
71
|
-
**(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
|
|
72
|
-
}.compact
|
|
73
|
-
end
|
|
74
|
-
|
|
53
|
+
class QualityReport
|
|
75
54
|
##
|
|
76
55
|
# @param audit [Html2rss::AutoSource::Cleanup::AuditResult]
|
|
77
56
|
# @param native_feed [String, nil]
|
|
@@ -85,11 +64,18 @@ module Html2rss
|
|
|
85
64
|
end
|
|
86
65
|
new(warnings: report_warnings.freeze, metrics: audit.metrics, native_feed:, defer_reason:)
|
|
87
66
|
end
|
|
67
|
+
|
|
68
|
+
##
|
|
69
|
+
# @return [Hash{Symbol => Object}]
|
|
70
|
+
def to_h
|
|
71
|
+
{
|
|
72
|
+
warnings: warnings.map(&:to_s),
|
|
73
|
+
metrics:,
|
|
74
|
+
**(native_feed ? { native_feed:, defer_reason: defer_reason.to_s } : {})
|
|
75
|
+
}.compact
|
|
76
|
+
end
|
|
88
77
|
end
|
|
89
78
|
|
|
90
|
-
##
|
|
91
|
-
# Immutable outcome of a configuration test. Success carries +rss+ XML from the
|
|
92
|
-
# first live extraction; failures carry a typed {FailureKind}.
|
|
93
79
|
Result = Data.define(
|
|
94
80
|
:success,
|
|
95
81
|
:item_count,
|
|
@@ -98,13 +84,20 @@ module Html2rss
|
|
|
98
84
|
:channel_url,
|
|
99
85
|
:strategy_used,
|
|
100
86
|
:duration_seconds,
|
|
101
|
-
:
|
|
87
|
+
:validation_issues,
|
|
102
88
|
:error_message,
|
|
103
89
|
:failure_kind,
|
|
104
90
|
:rss,
|
|
105
91
|
:quality_report,
|
|
106
|
-
:enhance_compare
|
|
107
|
-
|
|
92
|
+
:enhance_compare,
|
|
93
|
+
:response_body
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
##
|
|
97
|
+
# Immutable outcome of a configuration test. Success carries +rss+ XML from the
|
|
98
|
+
# first live extraction; failures carry a typed {FailureKind}.
|
|
99
|
+
# +response_body+ is a copy of the fetched page for in-process readers and stays off {#to_h}.
|
|
100
|
+
class Result
|
|
108
101
|
##
|
|
109
102
|
# @param success [Boolean]
|
|
110
103
|
# @param item_count [Integer]
|
|
@@ -113,22 +106,23 @@ module Html2rss
|
|
|
113
106
|
# @param channel_url [String, nil]
|
|
114
107
|
# @param strategy_used [Symbol, nil]
|
|
115
108
|
# @param duration_seconds [Float]
|
|
116
|
-
# @param
|
|
109
|
+
# @param validation_issues [Array<Html2rss::Config::ValidationIssue>, nil]
|
|
117
110
|
# @param error_message [String, nil]
|
|
118
111
|
# @param failure_kind [FailureKind, nil]
|
|
119
112
|
# @param rss [String, nil]
|
|
120
113
|
# @param quality_report [QualityReport, nil]
|
|
121
114
|
# @param enhance_compare [Hash, nil]
|
|
115
|
+
# @param response_body [String, nil] fetched page HTML copied from the pipeline response
|
|
122
116
|
def initialize(success:, item_count:, sample_items:, channel_title:, channel_url:, # rubocop:disable Metrics/ParameterLists
|
|
123
|
-
strategy_used:, duration_seconds:,
|
|
124
|
-
failure_kind:, rss:, quality_report: nil, enhance_compare: nil)
|
|
117
|
+
strategy_used:, duration_seconds:, validation_issues:, error_message:,
|
|
118
|
+
failure_kind:, rss:, quality_report: nil, enhance_compare: nil, response_body: nil)
|
|
125
119
|
super
|
|
126
120
|
end
|
|
127
121
|
|
|
128
122
|
##
|
|
129
123
|
# @return [Boolean] whether the schema validation succeeded
|
|
130
124
|
def valid_schema?
|
|
131
|
-
|
|
125
|
+
validation_issues.nil? || validation_issues.empty?
|
|
132
126
|
end
|
|
133
127
|
|
|
134
128
|
##
|
|
@@ -138,8 +132,8 @@ module Html2rss
|
|
|
138
132
|
end
|
|
139
133
|
|
|
140
134
|
##
|
|
141
|
-
# @return [Hash{Symbol => Object}] hash
|
|
142
|
-
def to_h # rubocop:disable Metrics/MethodLength
|
|
135
|
+
# @return [Hash{Symbol => Object}] public test hash; +response_body+ stays off this wire
|
|
136
|
+
def to_h # rubocop:disable Metrics/MethodLength, Metrics/AbcSize -- wire serialization omits response_body
|
|
143
137
|
{
|
|
144
138
|
success:,
|
|
145
139
|
item_count:,
|
|
@@ -148,7 +142,7 @@ module Html2rss
|
|
|
148
142
|
channel_url:,
|
|
149
143
|
strategy_used:,
|
|
150
144
|
duration_seconds:,
|
|
151
|
-
|
|
145
|
+
validation_issues: validation_issues&.map(&:to_h),
|
|
152
146
|
error_message:,
|
|
153
147
|
failure_kind: failure_kind&.to_sym,
|
|
154
148
|
rss:,
|
|
@@ -169,92 +163,113 @@ module Html2rss
|
|
|
169
163
|
# @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
|
|
170
164
|
# @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
|
|
171
165
|
# @return [Html2rss::Test::Result]
|
|
172
|
-
def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
166
|
+
def call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
|
|
173
167
|
strict_quality: false, compare_enhance: false)
|
|
174
168
|
raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
|
|
175
|
-
return validation_failure_result(validation
|
|
169
|
+
return validation_failure_result(validation, raw_config) unless validation.success?
|
|
176
170
|
|
|
171
|
+
config = build_test_config(raw_config, strategy, params)
|
|
172
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
173
|
+
execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
|
|
174
|
+
rescue StandardError => error
|
|
175
|
+
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
176
|
+
execution_failure_result(error, raw_config, duration)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def build_test_config(raw_config, strategy, params)
|
|
177
180
|
raw_config[:strategy] = strategy.to_sym if strategy
|
|
178
181
|
raw_config[:params] = params if params&.any?
|
|
182
|
+
Config.from_hash(raw_config)
|
|
183
|
+
end
|
|
184
|
+
private_class_method :build_test_config
|
|
185
|
+
|
|
186
|
+
def compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
187
|
+
quality_report_for(
|
|
188
|
+
feed_result.to_rss.items,
|
|
189
|
+
channel_url: raw_config.dig(:channel, :url).to_s,
|
|
190
|
+
config:,
|
|
191
|
+
feed_result:,
|
|
192
|
+
pipeline_outcome:
|
|
193
|
+
)
|
|
194
|
+
end
|
|
195
|
+
private_class_method :compile_quality_report
|
|
179
196
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
strategy_used:,
|
|
216
|
-
duration_seconds: duration.round(3),
|
|
217
|
-
validation_errors: nil,
|
|
218
|
-
error_message:,
|
|
219
|
-
failure_kind:,
|
|
220
|
-
rss: passed ? rss_xml : nil,
|
|
221
|
-
quality_report:,
|
|
222
|
-
enhance_compare:
|
|
223
|
-
)
|
|
224
|
-
rescue StandardError => error
|
|
225
|
-
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
226
|
-
execution_failure_result(error, raw_config, duration)
|
|
227
|
-
end
|
|
197
|
+
def execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
|
|
198
|
+
compare_enhance:)
|
|
199
|
+
feed_result, pipeline_outcome = extract_feed(raw_config)
|
|
200
|
+
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
201
|
+
quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
|
|
202
|
+
enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
|
|
203
|
+
|
|
204
|
+
build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
|
|
205
|
+
min_items:, strict_quality:, response_body: pipeline_outcome&.response&.body)
|
|
206
|
+
end
|
|
207
|
+
private_class_method :execute_timed_pipeline
|
|
208
|
+
|
|
209
|
+
def build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
|
|
210
|
+
min_items:, strict_quality:, response_body:)
|
|
211
|
+
rss_doc = feed_result.to_rss
|
|
212
|
+
outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
|
|
213
|
+
strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
|
|
214
|
+
RequestService.default_strategy_name
|
|
215
|
+
|
|
216
|
+
Result.new(
|
|
217
|
+
success: outcome.passed,
|
|
218
|
+
item_count: rss_doc.items.size,
|
|
219
|
+
sample_items: extract_samples(rss_doc.items),
|
|
220
|
+
channel_title: feed_result.channel_title,
|
|
221
|
+
channel_url: raw_config.dig(:channel, :url).to_s,
|
|
222
|
+
strategy_used:,
|
|
223
|
+
duration_seconds: duration.round(3),
|
|
224
|
+
validation_issues: nil,
|
|
225
|
+
error_message: outcome.error_message,
|
|
226
|
+
failure_kind: outcome.failure_kind,
|
|
227
|
+
rss: outcome.passed ? rss_doc.to_s : nil,
|
|
228
|
+
quality_report:,
|
|
229
|
+
enhance_compare:,
|
|
230
|
+
response_body:
|
|
231
|
+
)
|
|
228
232
|
end
|
|
233
|
+
private_class_method :build_test_result
|
|
229
234
|
|
|
230
235
|
def extract_samples(items, limit: 3)
|
|
231
236
|
items.first(limit).map do |item|
|
|
237
|
+
article = feed_item_as_article(item)
|
|
232
238
|
{
|
|
233
|
-
title:
|
|
234
|
-
url:
|
|
235
|
-
published_at:
|
|
239
|
+
title: article.title.to_s.strip,
|
|
240
|
+
url: article.url.to_s,
|
|
241
|
+
published_at: article.published_at
|
|
236
242
|
}
|
|
237
243
|
end
|
|
238
244
|
end
|
|
239
245
|
private_class_method :extract_samples
|
|
240
246
|
|
|
247
|
+
def feed_item_as_article(item)
|
|
248
|
+
case item
|
|
249
|
+
when Article then item
|
|
250
|
+
else
|
|
251
|
+
Article.new(title: item.title, url: item.link, published_at: item.pubDate)
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
private_class_method :feed_item_as_article
|
|
255
|
+
|
|
241
256
|
##
|
|
242
257
|
# Builds a ship-quality report for RSS items (shared by test and MCP apply).
|
|
243
258
|
#
|
|
244
259
|
# @param items [Array]
|
|
245
260
|
# @param channel_url [String]
|
|
246
|
-
# @param
|
|
261
|
+
# @param config [Html2rss::Config]
|
|
247
262
|
# @param feed_result [Html2rss::FeedResult]
|
|
248
263
|
# @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
|
|
249
264
|
# @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
|
|
250
265
|
# @return [QualityReport]
|
|
251
|
-
def quality_report_for(items, channel_url:,
|
|
266
|
+
def quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
|
|
252
267
|
probe_native_feed: true)
|
|
253
268
|
audit = AutoSource::Cleanup.audit_feed_items(items)
|
|
254
|
-
native_feed = probe_native_feed ? probe_native_feed_url(channel_url,
|
|
269
|
+
native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
|
|
255
270
|
report = QualityReport.from_audit(audit, native_feed:)
|
|
256
271
|
report = append_url_mismatch_warning(report, channel_url, feed_result)
|
|
257
|
-
merge_enhance_audit(report,
|
|
272
|
+
merge_enhance_audit(report, config, pipeline_outcome)
|
|
258
273
|
end
|
|
259
274
|
|
|
260
275
|
def extract_feed(raw_config)
|
|
@@ -263,18 +278,15 @@ module Html2rss
|
|
|
263
278
|
end
|
|
264
279
|
private_class_method :extract_feed
|
|
265
280
|
|
|
266
|
-
def enhance_enabled?(
|
|
267
|
-
config = Config.from_hash(raw_config)
|
|
281
|
+
def enhance_enabled?(config)
|
|
268
282
|
return false unless config.selectors
|
|
269
283
|
|
|
270
284
|
!!config.selectors.dig(:items, :enhance)
|
|
271
285
|
end
|
|
272
286
|
private_class_method :enhance_enabled?
|
|
273
287
|
|
|
274
|
-
def build_enhance_compare(
|
|
288
|
+
def build_enhance_compare(config, pipeline_outcome)
|
|
275
289
|
return nil unless pipeline_outcome
|
|
276
|
-
|
|
277
|
-
config = Config.from_hash(raw_config)
|
|
278
290
|
return nil unless config.selectors
|
|
279
291
|
|
|
280
292
|
EnhanceAudit.compare(
|
|
@@ -285,10 +297,8 @@ module Html2rss
|
|
|
285
297
|
end
|
|
286
298
|
private_class_method :build_enhance_compare
|
|
287
299
|
|
|
288
|
-
def merge_enhance_audit(report,
|
|
289
|
-
return report unless pipeline_outcome && enhance_enabled?(
|
|
290
|
-
|
|
291
|
-
config = Config.from_hash(raw_config)
|
|
300
|
+
def merge_enhance_audit(report, config, pipeline_outcome) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- warning merge + metrics
|
|
301
|
+
return report unless pipeline_outcome && enhance_enabled?(config)
|
|
292
302
|
return report unless config.selectors
|
|
293
303
|
|
|
294
304
|
slice = EnhanceAudit.probe(
|
|
@@ -349,67 +359,11 @@ module Html2rss
|
|
|
349
359
|
end
|
|
350
360
|
private_class_method :url_mismatch?
|
|
351
361
|
|
|
352
|
-
def
|
|
353
|
-
metrics = quality_report.metrics
|
|
354
|
-
item_count = metrics[:item_count]
|
|
355
|
-
return false if item_count.zero?
|
|
356
|
-
|
|
357
|
-
duplicate_urls_failure?(metrics) || junk_title_ratio_failure?(metrics) || short_title_failure?(metrics)
|
|
358
|
-
end
|
|
359
|
-
private_class_method :quality_failure?
|
|
360
|
-
|
|
361
|
-
def duplicate_urls_failure?(metrics)
|
|
362
|
-
metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
|
|
363
|
-
end
|
|
364
|
-
private_class_method :duplicate_urls_failure?
|
|
365
|
-
|
|
366
|
-
def junk_title_ratio_failure?(metrics)
|
|
367
|
-
metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
|
|
368
|
-
end
|
|
369
|
-
private_class_method :junk_title_ratio_failure?
|
|
370
|
-
|
|
371
|
-
def short_title_failure?(metrics)
|
|
372
|
-
metrics[:short_title_count].positive?
|
|
373
|
-
end
|
|
374
|
-
private_class_method :short_title_failure?
|
|
375
|
-
|
|
376
|
-
def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
|
|
377
|
-
return [nil, nil] if min_items_passed && !quality_failed
|
|
378
|
-
unless min_items_passed
|
|
379
|
-
return [FailureKind.coerce(:min_items),
|
|
380
|
-
"Extracted #{item_count} items (minimum required: #{min_items})"]
|
|
381
|
-
end
|
|
382
|
-
|
|
383
|
-
[FailureKind.coerce(:quality), quality_failure_message(quality_report)]
|
|
384
|
-
end
|
|
385
|
-
private_class_method :outcome_failure
|
|
386
|
-
|
|
387
|
-
def quality_failure_message(quality_report)
|
|
388
|
-
reasons = quality_failure_reasons(quality_report.metrics)
|
|
389
|
-
"Feed quality check failed (#{reasons.join(', ')})"
|
|
390
|
-
end
|
|
391
|
-
private_class_method :quality_failure_message
|
|
392
|
-
|
|
393
|
-
def quality_failure_reasons(metrics)
|
|
394
|
-
reasons = []
|
|
395
|
-
reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
|
|
396
|
-
reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
|
|
397
|
-
reasons << 'short_titles' if short_title_failure?(metrics)
|
|
398
|
-
reasons
|
|
399
|
-
end
|
|
400
|
-
private_class_method :quality_failure_reasons
|
|
401
|
-
|
|
402
|
-
def log_strict_quality_failure(failure_kind, item_count)
|
|
403
|
-
Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
|
|
404
|
-
end
|
|
405
|
-
private_class_method :log_strict_quality_failure
|
|
406
|
-
|
|
407
|
-
def probe_native_feed_url(channel_url, raw_config)
|
|
362
|
+
def probe_native_feed_url(channel_url, config)
|
|
408
363
|
return nil if channel_url.to_s.empty?
|
|
409
364
|
|
|
410
|
-
config = Config.from_hash(raw_config)
|
|
411
365
|
resources = FeedPipeline::RuntimePolicy.resources_for(config)
|
|
412
|
-
strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(
|
|
366
|
+
strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(config.strategy)
|
|
413
367
|
session = RequestSession.build(
|
|
414
368
|
config:, strategy:, budget: resources.budget, policy: resources.policy
|
|
415
369
|
)
|
|
@@ -419,7 +373,7 @@ module Html2rss
|
|
|
419
373
|
end
|
|
420
374
|
private_class_method :probe_native_feed_url
|
|
421
375
|
|
|
422
|
-
def validation_failure_result(
|
|
376
|
+
def validation_failure_result(report, raw_config) # rubocop:disable Metrics/MethodLength
|
|
423
377
|
Result.new(
|
|
424
378
|
success: false,
|
|
425
379
|
item_count: 0,
|
|
@@ -428,9 +382,9 @@ module Html2rss
|
|
|
428
382
|
channel_url: raw_config.dig(:channel, :url),
|
|
429
383
|
strategy_used: raw_config[:strategy],
|
|
430
384
|
duration_seconds: 0.0,
|
|
431
|
-
|
|
385
|
+
validation_issues: report.issues,
|
|
432
386
|
error_message: 'Configuration schema validation failed',
|
|
433
|
-
failure_kind: FailureKind
|
|
387
|
+
failure_kind: FailureKind[:schema],
|
|
434
388
|
rss: nil,
|
|
435
389
|
quality_report: nil
|
|
436
390
|
)
|
|
@@ -446,9 +400,9 @@ module Html2rss
|
|
|
446
400
|
channel_url: raw_config.dig(:channel, :url),
|
|
447
401
|
strategy_used: raw_config[:strategy],
|
|
448
402
|
duration_seconds: duration.round(3),
|
|
449
|
-
|
|
403
|
+
validation_issues: nil,
|
|
450
404
|
error_message: "#{error.class}: #{error.message}",
|
|
451
|
-
failure_kind: FailureKind
|
|
405
|
+
failure_kind: FailureKind[:execution],
|
|
452
406
|
rss: nil,
|
|
453
407
|
quality_report: nil
|
|
454
408
|
)
|
data/lib/html2rss/url.rb
CHANGED
|
@@ -42,14 +42,21 @@ module Html2rss
|
|
|
42
42
|
url = Addressable::URI.parse(relative_url.to_s.strip)
|
|
43
43
|
return new(url) if url.absolute?
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
base_uri.path = '/' if base_uri.path.empty?
|
|
47
|
-
|
|
48
|
-
new(base_uri.join(url).normalize)
|
|
45
|
+
base_url.is_a?(self) ? base_url.join_uri(url) : join_parsed_base(base_url, url)
|
|
49
46
|
rescue Addressable::URI::InvalidURIError
|
|
50
47
|
raise ArgumentError, 'URL could not be parsed'
|
|
51
48
|
end
|
|
52
49
|
|
|
50
|
+
# @param base_url [String]
|
|
51
|
+
# @param url [Addressable::URI]
|
|
52
|
+
# @return [Url]
|
|
53
|
+
def self.join_parsed_base(base_url, url)
|
|
54
|
+
uri = Addressable::URI.parse(base_url.to_s)
|
|
55
|
+
uri.path = '/' if uri.path.empty?
|
|
56
|
+
new(uri.join(url).normalize)
|
|
57
|
+
end
|
|
58
|
+
private_class_method :join_parsed_base
|
|
59
|
+
|
|
53
60
|
##
|
|
54
61
|
# Creates a URL by sanitizing a raw URL string.
|
|
55
62
|
# Removes spaces and extracts the first valid URL from the string.
|
|
@@ -305,6 +312,20 @@ module Html2rss
|
|
|
305
312
|
# @return [String] the debug representation
|
|
306
313
|
def inspect = "#<#{self.class}:#{object_id} @uri=#{@uri.inspect}>"
|
|
307
314
|
|
|
315
|
+
##
|
|
316
|
+
# Resolves +relative_uri+ against this URL's already-parsed Addressable URI.
|
|
317
|
+
#
|
|
318
|
+
# @param relative_uri [Addressable::URI]
|
|
319
|
+
# @return [Url]
|
|
320
|
+
def join_uri(relative_uri)
|
|
321
|
+
base = @uri
|
|
322
|
+
if base.path.empty?
|
|
323
|
+
base = base.dup
|
|
324
|
+
base.path = '/'
|
|
325
|
+
end
|
|
326
|
+
self.class.new(base.join(relative_uri).normalize)
|
|
327
|
+
end
|
|
328
|
+
|
|
308
329
|
protected
|
|
309
330
|
|
|
310
331
|
# Scheme, authority, slash-stripped path, and query (fragment omitted).
|
data/lib/html2rss/version.rb
CHANGED
data/lib/html2rss.rb
CHANGED
|
@@ -11,20 +11,30 @@ loader.inflector.inflect(
|
|
|
11
11
|
loader.setup
|
|
12
12
|
|
|
13
13
|
require 'logger'
|
|
14
|
-
require 'forwardable'
|
|
15
14
|
require 'html2rss/defaults'
|
|
16
15
|
|
|
17
16
|
##
|
|
18
17
|
# The Html2rss namespace.
|
|
19
18
|
module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
20
19
|
##
|
|
21
|
-
#
|
|
20
|
+
# Forwards debug/info/warn/error to {Html2rss.logger}.
|
|
22
21
|
module Log
|
|
23
22
|
class << self
|
|
24
|
-
|
|
23
|
+
##
|
|
24
|
+
# @return [true]
|
|
25
|
+
def debug(...) = Html2rss.logger.debug(...)
|
|
25
26
|
|
|
26
|
-
|
|
27
|
-
|
|
27
|
+
##
|
|
28
|
+
# @return [true]
|
|
29
|
+
def info(...) = Html2rss.logger.info(...)
|
|
30
|
+
|
|
31
|
+
##
|
|
32
|
+
# @return [true]
|
|
33
|
+
def warn(...) = Html2rss.logger.warn(...)
|
|
34
|
+
|
|
35
|
+
##
|
|
36
|
+
# @return [true]
|
|
37
|
+
def error(...) = Html2rss.logger.error(...)
|
|
28
38
|
end
|
|
29
39
|
end
|
|
30
40
|
|
|
@@ -33,7 +43,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
33
43
|
# Golden path step 1 (optional); use {.recon} for verdict and native_feed.
|
|
34
44
|
#
|
|
35
45
|
# @param url [String] source page URL
|
|
36
|
-
# @param strategy [Symbol] request strategy (:auto, :
|
|
46
|
+
# @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
|
|
37
47
|
# @param deep [Boolean] when true and strategy is :auto, one Botasaurus hop if configured
|
|
38
48
|
# @return [Html2rss::PageRecon::Diagnostics::Report]
|
|
39
49
|
def self.inspect(url, strategy: :auto, deep: false, **)
|
|
@@ -45,7 +55,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
45
55
|
# Golden path step 2 (optional); adds verdict beyond {.inspect}.
|
|
46
56
|
#
|
|
47
57
|
# @param url [String, Html2rss::Url] source page URL
|
|
48
|
-
# @param strategy [Symbol] request strategy (:auto, :
|
|
58
|
+
# @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
|
|
49
59
|
# @return [Html2rss::Recon::Result]
|
|
50
60
|
def self.recon(url, strategy: :auto, **)
|
|
51
61
|
Recon.call(url, strategy:, **)
|
|
@@ -56,7 +66,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
56
66
|
# Golden path step 3.
|
|
57
67
|
#
|
|
58
68
|
# @param url [String] source page URL
|
|
59
|
-
# @param strategy [Symbol] request strategy (+:auto+, +:
|
|
69
|
+
# @param strategy [Symbol] request strategy (+:auto+, +:default+, +:botasaurus+)
|
|
60
70
|
# @option options [String, nil] :items_selector optional CSS selector hint for items
|
|
61
71
|
# @option options [Array<String>, nil] :topics optional directory topics override
|
|
62
72
|
# @option options [String, nil] :title optional title override
|
|
@@ -79,7 +89,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
79
89
|
# @param config_input [Hash, String]
|
|
80
90
|
# @param feed_name [String, nil]
|
|
81
91
|
# @param params [Hash]
|
|
82
|
-
# @return [
|
|
92
|
+
# @return [Html2rss::Config::ValidationReport]
|
|
83
93
|
def self.validate(config_input, feed_name = nil, params: {})
|
|
84
94
|
_raw, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
|
|
85
95
|
validation
|
|
@@ -140,7 +150,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
140
150
|
# Scrapes multiple URLs in parallel using auto-source article discovery.
|
|
141
151
|
#
|
|
142
152
|
# @param urls [Enumerable<String>] list of URLs to scrape
|
|
143
|
-
# @param strategy [Symbol] request strategy (:auto, :
|
|
153
|
+
# @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
|
|
144
154
|
# @param limit [Integer] max articles to keep per URL (default: 10)
|
|
145
155
|
# @param concurrency [Integer] max worker threads (default: 5)
|
|
146
156
|
# @return [Html2rss::Batch::BatchResult]
|
|
@@ -152,7 +162,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
152
162
|
# Inspects multiple URLs in parallel with per-URL error isolation.
|
|
153
163
|
#
|
|
154
164
|
# @param urls [Enumerable<String>] list of URLs to inspect
|
|
155
|
-
# @param strategy [Symbol] request strategy (:auto, :
|
|
165
|
+
# @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
|
|
156
166
|
# @param concurrency [Integer] max worker threads (default: 5)
|
|
157
167
|
# @return [Html2rss::Batch::BatchResult]
|
|
158
168
|
def self.batch_inspect(urls, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
|
|
@@ -163,7 +173,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
163
173
|
# Runs recon across multiple URLs in parallel with per-URL error isolation.
|
|
164
174
|
#
|
|
165
175
|
# @param urls [Enumerable<String>] list of URLs to recon
|
|
166
|
-
# @param strategy [Symbol] request strategy (:auto, :
|
|
176
|
+
# @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
|
|
167
177
|
# @param concurrency [Integer] max worker threads (default: 5)
|
|
168
178
|
# @option options [String, nil] :cache_dir optional HTML cache directory
|
|
169
179
|
# @return [Html2rss::Batch::BatchResult]
|
|
@@ -179,7 +189,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
179
189
|
# @param pretty [Boolean] whether to pretty-print JSON
|
|
180
190
|
# @return [String]
|
|
181
191
|
def self.schema_json(pretty: true)
|
|
182
|
-
Config.json_schema_json(pretty:)
|
|
192
|
+
Config::Schema.json_schema_json(pretty:)
|
|
183
193
|
end
|
|
184
194
|
|
|
185
195
|
##
|
|
@@ -319,26 +329,24 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
319
329
|
end
|
|
320
330
|
|
|
321
331
|
##
|
|
322
|
-
# @return [
|
|
332
|
+
# @return [Logger] the logger
|
|
323
333
|
def logger
|
|
324
334
|
defaults.logger
|
|
325
335
|
end
|
|
326
336
|
|
|
327
337
|
##
|
|
328
|
-
# @param logger [
|
|
338
|
+
# @param logger [Logger] the new logger
|
|
329
339
|
def logger=(logger)
|
|
330
340
|
configure { |config| config.logger = logger }
|
|
331
341
|
end
|
|
332
342
|
|
|
333
|
-
private
|
|
334
|
-
|
|
335
343
|
##
|
|
336
344
|
# Resets the global defaults (mainly for testing).
|
|
337
345
|
#
|
|
338
346
|
# @return [void]
|
|
339
347
|
def reset_defaults!
|
|
340
348
|
@defaults = nil
|
|
341
|
-
logger.level = defaults.log_level
|
|
349
|
+
logger.level = defaults.log_level
|
|
342
350
|
end
|
|
343
351
|
end
|
|
344
352
|
|
|
@@ -362,7 +370,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
|
|
|
362
370
|
end
|
|
363
371
|
end
|
|
364
372
|
|
|
365
|
-
logger.level = defaults.log_level
|
|
373
|
+
logger.level = defaults.log_level
|
|
366
374
|
end
|
|
367
375
|
|
|
368
376
|
loader.eager_load
|
|
@@ -7,10 +7,10 @@ require_relative '../html2rss'
|
|
|
7
7
|
namespace :config do
|
|
8
8
|
desc 'Generate config JSON schema'
|
|
9
9
|
task :schema do
|
|
10
|
-
destination = Html2rss::Config.
|
|
10
|
+
destination = Html2rss::Config::Schema.path
|
|
11
11
|
|
|
12
12
|
FileUtils.mkdir_p(File.dirname(destination))
|
|
13
|
-
File.write(destination, "#{Html2rss::Config.json_schema_json}\n")
|
|
13
|
+
File.write(destination, "#{Html2rss::Config::Schema.json_schema_json}\n")
|
|
14
14
|
|
|
15
15
|
puts "Generated config schema at #{destination}"
|
|
16
16
|
end
|