html2rss 0.32.4 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/html2rss/article/enclosure.rb +2 -6
- data/lib/html2rss/auto_source/cleanup.rb +1 -3
- data/lib/html2rss/auto_source/scraper/html.rb +2 -8
- data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
- data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
- data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
- data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
- data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
- data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
- data/lib/html2rss/auto_source/scraper.rb +0 -11
- data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
- data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
- data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
- data/lib/html2rss/auto_source.rb +1 -1
- data/lib/html2rss/batch.rb +22 -32
- data/lib/html2rss/capture/README.md +1 -1
- data/lib/html2rss/capture.rb +7 -27
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +33 -27
- data/lib/html2rss/config/request_controls.rb +10 -2
- data/lib/html2rss/config/schema.rb +8 -10
- data/lib/html2rss/config/selectors_validator.rb +1 -0
- data/lib/html2rss/config/validator.rb +3 -3
- data/lib/html2rss/config.rb +1 -1
- data/lib/html2rss/defaults.rb +5 -16
- data/lib/html2rss/doctor/botasaurus.rb +11 -19
- data/lib/html2rss/error.rb +7 -3
- data/lib/html2rss/feed_builder/rss.rb +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +3 -3
- data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +3 -3
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +5 -9
- data/lib/html2rss/feed_result.rb +4 -3
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
- data/lib/html2rss/html/probe.rb +51 -51
- data/lib/html2rss/link_destination/path_classifier.rb +11 -33
- data/lib/html2rss/mcp/config_argument.rb +9 -2
- data/lib/html2rss/mcp/contract.rb +7 -5
- data/lib/html2rss/mcp/outcome.rb +4 -5
- data/lib/html2rss/mcp/runtime.rb +17 -24
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/diagnostics.rb +7 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +10 -12
- data/lib/html2rss/request_service/blocked_surface.rb +3 -7
- data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +1 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +0 -1
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +8 -26
- data/lib/html2rss/request_service/response.rb +3 -6
- data/lib/html2rss/request_session/pager/base.rb +3 -3
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/scoring/engine.rb +4 -32
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +14 -18
- data/lib/html2rss/sst/node.rb +9 -3
- data/lib/html2rss/sst/normalizer.rb +25 -24
- data/lib/html2rss/sst/text.rb +53 -57
- data/lib/html2rss/status.rb +23 -30
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/discovery.rb +117 -187
- data/lib/html2rss/syndication/parser.rb +92 -115
- data/lib/html2rss/test/enhance_audit.rb +13 -15
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +29 -48
- data/lib/html2rss/url.rb +53 -53
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +2 -1
- metadata +4 -2
data/lib/html2rss/cli/render.rb
CHANGED
|
@@ -10,116 +10,117 @@ module Html2rss
|
|
|
10
10
|
# ANSI colors for recon verdict labels in text output.
|
|
11
11
|
VERDICT_COLORS = { build: "\e[32m", defer: "\e[33m" }.freeze
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
13
|
+
class << self
|
|
14
|
+
##
|
|
15
|
+
# @param results [Array<Hash>]
|
|
16
|
+
# @param format [String]
|
|
17
|
+
# @param batch_mode [Boolean]
|
|
18
|
+
# @return [void]
|
|
19
|
+
def inspect_output(results, format:, batch_mode:)
|
|
20
|
+
if format == 'json'
|
|
21
|
+
json(results, batch_mode)
|
|
22
|
+
else
|
|
23
|
+
results.each { |data| probe_card(ProbeView.from_wire(data)) }
|
|
24
|
+
end
|
|
25
25
|
end
|
|
26
|
-
end
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
27
|
+
##
|
|
28
|
+
# @param results [Array<Html2rss::Recon::Result>]
|
|
29
|
+
# @param format [String]
|
|
30
|
+
# @param batch_mode [Boolean]
|
|
31
|
+
# @param url_only [Boolean]
|
|
32
|
+
# @return [void]
|
|
33
|
+
def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
34
|
+
if url_only
|
|
35
|
+
results.each { |r| puts r.requested_url }
|
|
36
|
+
elsif format == 'json'
|
|
37
|
+
json(results.map(&:to_h), batch_mode)
|
|
38
|
+
elsif format == 'tsv'
|
|
39
|
+
puts %w[verdict status requested_url final_url native_feed notes].join("\t")
|
|
40
|
+
results.each do |r|
|
|
41
|
+
row = [
|
|
42
|
+
r.verdict.to_s.upcase,
|
|
43
|
+
r.status || '-',
|
|
44
|
+
r.requested_url,
|
|
45
|
+
r.final_url,
|
|
46
|
+
r.native_feed || '-',
|
|
47
|
+
r.notes.join('; ')
|
|
48
|
+
]
|
|
49
|
+
puts row.join("\t")
|
|
50
|
+
end
|
|
51
|
+
else
|
|
52
|
+
results.each { |r| probe_card(ProbeView.from_recon(r)) }
|
|
51
53
|
end
|
|
52
|
-
else
|
|
53
|
-
results.each { |r| probe_card(ProbeView.from_recon(r)) }
|
|
54
54
|
end
|
|
55
|
-
end
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
56
|
+
##
|
|
57
|
+
# @param view [ProbeView]
|
|
58
|
+
# @return [void]
|
|
59
|
+
def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
60
|
+
if view.verdict
|
|
61
|
+
color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
|
|
62
|
+
puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
|
|
63
|
+
else
|
|
64
|
+
puts view.requested
|
|
65
|
+
end
|
|
66
|
+
probe_lines(view)
|
|
67
|
+
if view.alternate_feeds.any?
|
|
68
|
+
hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
|
|
69
|
+
puts " Feeds: #{hrefs.join(', ')}" if hrefs.any?
|
|
70
|
+
end
|
|
71
|
+
puts " Feed: #{view.native_feed}" if view.native_feed
|
|
72
|
+
puts " Notes: #{view.notes.join(', ')}" if view.notes.any?
|
|
73
|
+
puts " Strategy: #{view.strategy}" if view.strategy
|
|
74
|
+
puts ''
|
|
71
75
|
end
|
|
72
|
-
puts " Feed: #{view.native_feed}" if view.native_feed
|
|
73
|
-
puts " Notes: #{view.notes.join(', ')}" if view.notes.any?
|
|
74
|
-
puts " Strategy: #{view.strategy}" if view.strategy
|
|
75
|
-
puts ''
|
|
76
|
-
end
|
|
77
76
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
77
|
+
##
|
|
78
|
+
# @param result [Html2rss::Test::Result]
|
|
79
|
+
# @param source [String]
|
|
80
|
+
# @return [void]
|
|
81
|
+
def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
82
|
+
if result.success
|
|
83
|
+
puts "\e[32m✓ Schema valid\e[0m (#{source})"
|
|
84
|
+
dur = "#{result.duration_seconds}s"
|
|
85
|
+
puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
|
|
86
|
+
puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
|
|
87
|
+
if result.sample_items.any?
|
|
88
|
+
puts 'Sample items:'
|
|
89
|
+
result.sample_items.each_with_index do |item, i|
|
|
90
|
+
puts " #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
|
|
91
|
+
puts " #{item[:url]}"
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
else
|
|
95
|
+
$stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
|
|
96
|
+
$stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
|
|
97
|
+
result.validation_issues&.each do |issue|
|
|
98
|
+
$stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
|
|
93
99
|
end
|
|
94
|
-
end
|
|
95
|
-
else
|
|
96
|
-
$stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
|
|
97
|
-
$stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
|
|
98
|
-
result.validation_issues&.each do |issue|
|
|
99
|
-
$stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
|
|
100
100
|
end
|
|
101
101
|
end
|
|
102
|
-
end
|
|
103
102
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
103
|
+
##
|
|
104
|
+
# @param data [Object]
|
|
105
|
+
# @param batch_mode [Boolean]
|
|
106
|
+
# @return [void]
|
|
107
|
+
def json(data, batch_mode)
|
|
108
|
+
payload = batch_mode ? data : data.first
|
|
109
|
+
puts JSON.pretty_generate(payload)
|
|
110
|
+
end
|
|
112
111
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
112
|
+
private
|
|
113
|
+
|
|
114
|
+
##
|
|
115
|
+
# @param view [ProbeView]
|
|
116
|
+
# @return [void]
|
|
117
|
+
def probe_lines(view)
|
|
118
|
+
if view.final && view.final != view.requested
|
|
119
|
+
puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
|
|
120
|
+
end
|
|
121
|
+
puts " Surface: #{view.surface} (#{view.articles_count} articles)"
|
|
119
122
|
end
|
|
120
|
-
puts " Surface: #{view.surface} (#{view.articles_count} articles)"
|
|
121
123
|
end
|
|
122
|
-
private_class_method :probe_lines
|
|
123
124
|
end
|
|
124
125
|
end
|
|
125
126
|
end
|
|
@@ -5,15 +5,13 @@ module Html2rss
|
|
|
5
5
|
##
|
|
6
6
|
# Validate command: file/glob/stdin branching and multi-file reporting.
|
|
7
7
|
module Validate
|
|
8
|
-
module_function
|
|
9
|
-
|
|
10
8
|
##
|
|
11
9
|
# @param files [Array<String>]
|
|
12
10
|
# @param params [Hash]
|
|
13
11
|
# @param quiet [Boolean]
|
|
14
12
|
# @return [void]
|
|
15
13
|
# @raise [Thor::Error]
|
|
16
|
-
def run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
14
|
+
def self.run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
17
15
|
if files.size == 2 && File.file?(files[0].to_s) && !File.exist?(files[1].to_s)
|
|
18
16
|
raise Thor::Error, "No such file: #{files[1]}" if path_like_config?(files[1])
|
|
19
17
|
|
|
@@ -45,7 +43,7 @@ module Html2rss
|
|
|
45
43
|
##
|
|
46
44
|
# @param files [Array<String>]
|
|
47
45
|
# @return [Array<String>]
|
|
48
|
-
def resolve_files(files)
|
|
46
|
+
def self.resolve_files(files)
|
|
49
47
|
return ['-'] if files.empty? || files == ['-']
|
|
50
48
|
|
|
51
49
|
resolved = []
|
data/lib/html2rss/cli.rb
CHANGED
|
@@ -114,24 +114,26 @@ module Html2rss
|
|
|
114
114
|
method_option :explain, type: :boolean, desc: 'Print capture diagnostics to stderr', default: false
|
|
115
115
|
# @param target [String, nil] URL or local HTML file
|
|
116
116
|
# @return [void]
|
|
117
|
-
def capture(target = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
117
|
+
def capture(target = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
118
118
|
strategy, local_file_path, url = prepare_auto_inputs(target, options[:input])
|
|
119
119
|
topics = options[:topics]&.split(',')&.map(&:strip)
|
|
120
120
|
|
|
121
|
-
result =
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
121
|
+
result = execute_feed do
|
|
122
|
+
Html2rss::Capture.build(
|
|
123
|
+
url,
|
|
124
|
+
strategy:,
|
|
125
|
+
items_selector: options[:items_selector],
|
|
126
|
+
topics:,
|
|
127
|
+
title: options[:title],
|
|
128
|
+
summary: options[:summary],
|
|
129
|
+
force: options[:force],
|
|
130
|
+
enhance: options[:enhance],
|
|
131
|
+
limit: options[:limit],
|
|
132
|
+
max_redirects: options[:max_redirects],
|
|
133
|
+
max_requests: options[:max_requests],
|
|
134
|
+
local_file_path:
|
|
135
|
+
)
|
|
136
|
+
end
|
|
135
137
|
|
|
136
138
|
if result.native_feed? && !options[:force]
|
|
137
139
|
$stderr.puts "First-party RSS/Atom feed detected at #{result.native_feed}. Use --force to capture anyway." # rubocop:disable Style/StderrPuts
|
|
@@ -158,15 +160,17 @@ module Html2rss
|
|
|
158
160
|
# @return [void]
|
|
159
161
|
def test(config_input = nil, feed_name = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
160
162
|
raw_content, input_source = read_config_input(config_input)
|
|
161
|
-
result =
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
163
|
+
result = execute_feed do
|
|
164
|
+
Html2rss.test(
|
|
165
|
+
raw_content,
|
|
166
|
+
feed_name,
|
|
167
|
+
min_items: options.fetch(:min_items, 1),
|
|
168
|
+
params: options[:params] || {},
|
|
169
|
+
strategy: options[:strategy],
|
|
170
|
+
strict_quality: options.fetch(:strict_quality, false),
|
|
171
|
+
compare_enhance: options.fetch(:compare_enhance, false)
|
|
172
|
+
)
|
|
173
|
+
end
|
|
170
174
|
|
|
171
175
|
if options[:json]
|
|
172
176
|
puts JSON.pretty_generate(result.to_h)
|
|
@@ -202,7 +206,7 @@ module Html2rss
|
|
|
202
206
|
Config.from_yaml(raw_content)
|
|
203
207
|
end
|
|
204
208
|
config[:params] = options[:params] || {}
|
|
205
|
-
apply_runtime_request_overrides!(config)
|
|
209
|
+
execute_feed { apply_runtime_request_overrides!(config) }
|
|
206
210
|
apply_local_file_input!(config, options[:input]) if options[:input]
|
|
207
211
|
|
|
208
212
|
run_feed_command { Html2rss.apply(config) }
|
|
@@ -421,7 +425,9 @@ module Html2rss
|
|
|
421
425
|
Html2rss::RequestService::BotasaurusConnectionFailed,
|
|
422
426
|
Html2rss::RequestService::BotasaurusServiceError,
|
|
423
427
|
Html2rss::RequestService::BlockedSurfaceDetected,
|
|
424
|
-
Html2rss::NoFeedItemsExtracted
|
|
428
|
+
Html2rss::NoFeedItemsExtracted,
|
|
429
|
+
Html2rss::InvalidOption,
|
|
430
|
+
Html2rss::Url::InvalidUrlError => error
|
|
425
431
|
raise Thor::Error, error.message
|
|
426
432
|
end
|
|
427
433
|
|
|
@@ -433,7 +439,7 @@ module Html2rss
|
|
|
433
439
|
max_redirects: options[:max_redirects],
|
|
434
440
|
max_requests: options[:max_requests],
|
|
435
441
|
local_file_path:,
|
|
436
|
-
limit: options[:limit]
|
|
442
|
+
limit: options[:limit],
|
|
437
443
|
keep_different_domain: options[:keep_different_domain]
|
|
438
444
|
)
|
|
439
445
|
end
|
|
@@ -52,6 +52,8 @@ module Html2rss
|
|
|
52
52
|
# @param max_requests [Integer, nil]
|
|
53
53
|
# @return [RequestControls]
|
|
54
54
|
def self.from_shortcut(strategy:, max_redirects: nil, max_requests: nil)
|
|
55
|
+
strategy = Option.strategy(strategy) unless strategy.nil?
|
|
56
|
+
check_budgets!(max_redirects, max_requests)
|
|
55
57
|
keys = []
|
|
56
58
|
keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
|
|
57
59
|
keys << :max_redirects unless max_redirects.nil?
|
|
@@ -68,7 +70,8 @@ module Html2rss
|
|
|
68
70
|
# @param max_requests [Integer, nil]
|
|
69
71
|
# @return [RequestControls]
|
|
70
72
|
def self.from_cli_options(strategy:, max_redirects: nil, max_requests: nil)
|
|
71
|
-
strategy_sym = strategy
|
|
73
|
+
strategy_sym = Option.strategy(strategy) if strategy
|
|
74
|
+
check_budgets!(max_redirects, max_requests)
|
|
72
75
|
keys = []
|
|
73
76
|
keys << :strategy if strategy
|
|
74
77
|
keys << :max_redirects unless max_redirects.nil?
|
|
@@ -76,7 +79,12 @@ module Html2rss
|
|
|
76
79
|
new(strategy: strategy_sym, max_redirects:, max_requests:, explicit_keys: keys)
|
|
77
80
|
end
|
|
78
81
|
|
|
79
|
-
|
|
82
|
+
def self.check_budgets!(max_redirects, max_requests)
|
|
83
|
+
Option.integer(:max_redirects, max_redirects, min: 0, optional: true)
|
|
84
|
+
Option.integer(:max_requests, max_requests, min: 1, optional: true)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private_class_method :explicit_keys_for, :request_value_for, :request_key?, :check_budgets!
|
|
80
88
|
|
|
81
89
|
##
|
|
82
90
|
# @param strategy [Symbol, nil] effective request strategy
|
|
@@ -7,8 +7,6 @@ module Html2rss
|
|
|
7
7
|
##
|
|
8
8
|
# JSON export adapter from the runtime validators to the packaged schema artifact.
|
|
9
9
|
module Schema
|
|
10
|
-
module_function
|
|
11
|
-
|
|
12
10
|
# Canonical filename for the exported config JSON schema artifact.
|
|
13
11
|
SCHEMA_FILENAME = 'html2rss-config.schema.json'
|
|
14
12
|
|
|
@@ -16,7 +14,7 @@ module Html2rss
|
|
|
16
14
|
# Returns the exported configuration JSON Schema.
|
|
17
15
|
#
|
|
18
16
|
# @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
|
|
19
|
-
def json_schema
|
|
17
|
+
def self.json_schema
|
|
20
18
|
load_json_schema_extension!
|
|
21
19
|
Builder.call
|
|
22
20
|
end
|
|
@@ -26,7 +24,7 @@ module Html2rss
|
|
|
26
24
|
#
|
|
27
25
|
# @param pretty [Boolean] whether to pretty-print the JSON output
|
|
28
26
|
# @return [String] serialized JSON Schema
|
|
29
|
-
def json_schema_json(pretty: true)
|
|
27
|
+
def self.json_schema_json(pretty: true)
|
|
30
28
|
pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
|
|
31
29
|
end
|
|
32
30
|
|
|
@@ -34,7 +32,7 @@ module Html2rss
|
|
|
34
32
|
# Resolves the packaged schema path used by downstream tools.
|
|
35
33
|
#
|
|
36
34
|
# @return [String] absolute path to the packaged JSON schema file
|
|
37
|
-
def path
|
|
35
|
+
def self.path
|
|
38
36
|
search_path = File.expand_path(__dir__)
|
|
39
37
|
|
|
40
38
|
loop do
|
|
@@ -51,7 +49,7 @@ module Html2rss
|
|
|
51
49
|
end
|
|
52
50
|
|
|
53
51
|
# @return [void]
|
|
54
|
-
def load_json_schema_extension!
|
|
52
|
+
def self.load_json_schema_extension!
|
|
55
53
|
require 'dry/schema/extensions/json_schema'
|
|
56
54
|
Dry::Schema.load_extensions(:json_schema)
|
|
57
55
|
end
|
|
@@ -221,7 +219,7 @@ module Html2rss
|
|
|
221
219
|
#
|
|
222
220
|
# @param schema [Hash] dry-schema JSON schema fragment for a selector
|
|
223
221
|
# @return [Hash] same schema with registry $ref wiring applied
|
|
224
|
-
def apply_selector_registry_refs!(schema)
|
|
222
|
+
def self.apply_selector_registry_refs!(schema)
|
|
225
223
|
properties = schema.fetch(:properties)
|
|
226
224
|
properties[:extractor] = extractor_wire_schema
|
|
227
225
|
properties[:post_process] = post_process_wire_schema
|
|
@@ -229,7 +227,7 @@ module Html2rss
|
|
|
229
227
|
end
|
|
230
228
|
|
|
231
229
|
# @return [Hash{Symbol => Object}]
|
|
232
|
-
def extractor_wire_schema
|
|
230
|
+
def self.extractor_wire_schema
|
|
233
231
|
{
|
|
234
232
|
description: 'Extractor used to pull a value from the selected element.',
|
|
235
233
|
oneOf: registry_ref_list('extractors', Selectors::Extractors::NAME_TO_CLASS)
|
|
@@ -237,7 +235,7 @@ module Html2rss
|
|
|
237
235
|
end
|
|
238
236
|
|
|
239
237
|
# @return [Hash{Symbol => Object}]
|
|
240
|
-
def post_process_wire_schema
|
|
238
|
+
def self.post_process_wire_schema
|
|
241
239
|
{
|
|
242
240
|
type: 'array',
|
|
243
241
|
description: 'Ordered transforms applied to the extracted value.',
|
|
@@ -251,7 +249,7 @@ module Html2rss
|
|
|
251
249
|
# @param catalog [String] $defs catalog name (`extractors` or `post_processors`)
|
|
252
250
|
# @param registry [Hash{Symbol => Class}]
|
|
253
251
|
# @return [Array<Hash>] oneOf entries of `{ '$ref' => ... }`
|
|
254
|
-
def registry_ref_list(catalog, registry)
|
|
252
|
+
def self.registry_ref_list(catalog, registry)
|
|
255
253
|
registry.keys.sort.map do |name|
|
|
256
254
|
{ '$ref' => "#/$defs/#{catalog}/#{name}" }
|
|
257
255
|
end
|
|
@@ -216,6 +216,7 @@ module Html2rss
|
|
|
216
216
|
def array_selector_shape_error(key, selector)
|
|
217
217
|
return "`#{key}` must be an array" unless selector.is_a?(Array)
|
|
218
218
|
return "`#{key}` must contain at least one element" if selector.empty?
|
|
219
|
+
return "`#{key}` must contain only selector names" unless selector.all? { _1 in String | Symbol }
|
|
219
220
|
|
|
220
221
|
nil
|
|
221
222
|
end
|
|
@@ -158,15 +158,15 @@ module Html2rss
|
|
|
158
158
|
if (url_string = values.dig(:channel, :url)) && !url_string.empty?
|
|
159
159
|
begin
|
|
160
160
|
Html2rss::Url.for_channel(url_string)
|
|
161
|
-
rescue
|
|
161
|
+
rescue Html2rss::Url::InvalidUrlError => error
|
|
162
162
|
key(%i[channel url]).failure(error.message)
|
|
163
163
|
end
|
|
164
164
|
end
|
|
165
165
|
|
|
166
166
|
if (image_string = values.dig(:channel, :image)) && !image_string.empty?
|
|
167
167
|
begin
|
|
168
|
-
Html2rss::Url.
|
|
169
|
-
rescue
|
|
168
|
+
Html2rss::Url.from_absolute_web(image_string)
|
|
169
|
+
rescue Html2rss::Url::InvalidUrlError => error
|
|
170
170
|
key(%i[channel image]).failure(error.message)
|
|
171
171
|
end
|
|
172
172
|
end
|
data/lib/html2rss/config.rb
CHANGED
|
@@ -131,7 +131,7 @@ module Html2rss
|
|
|
131
131
|
def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil,
|
|
132
132
|
keep_different_domain: nil)
|
|
133
133
|
auto_source = AutoSource::DEFAULT_CONFIG
|
|
134
|
-
auto_source = auto_source.merge(limit:) unless limit.nil?
|
|
134
|
+
auto_source = auto_source.merge(limit: Option.integer(:limit, limit, min: 1)) unless limit.nil?
|
|
135
135
|
unless keep_different_domain.nil?
|
|
136
136
|
auto_source = auto_source.merge(
|
|
137
137
|
cleanup: auto_source[:cleanup].merge(keep_different_domain:)
|
data/lib/html2rss/defaults.rb
CHANGED
|
@@ -57,7 +57,7 @@ module Html2rss
|
|
|
57
57
|
# @return [Logger] the logger
|
|
58
58
|
# @raise [ArgumentError] if logger is not a Logger
|
|
59
59
|
def logger=(logger)
|
|
60
|
-
raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(
|
|
60
|
+
raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(Logger)
|
|
61
61
|
|
|
62
62
|
@logger = logger
|
|
63
63
|
@logger.level = @log_level
|
|
@@ -114,20 +114,9 @@ module Html2rss
|
|
|
114
114
|
#
|
|
115
115
|
# @param strategy [Symbol, String, nil] the strategy plan name
|
|
116
116
|
# @return [Symbol, nil] the normalized strategy plan name
|
|
117
|
-
# @raise [
|
|
117
|
+
# @raise [InvalidOption] if the strategy plan is unknown
|
|
118
118
|
def default_strategy=(strategy)
|
|
119
|
-
|
|
120
|
-
@default_strategy = nil
|
|
121
|
-
else
|
|
122
|
-
unless strategy.is_a?(Symbol) || strategy.is_a?(String)
|
|
123
|
-
raise ArgumentError, 'strategy must be a Symbol or String'
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
normalized = strategy.to_sym
|
|
127
|
-
raise ArgumentError, "unknown strategy: #{strategy}" unless FeedPipeline::StrategyPlan.valid?(normalized)
|
|
128
|
-
|
|
129
|
-
@default_strategy = normalized
|
|
130
|
-
end
|
|
119
|
+
@default_strategy = (Option.strategy(strategy) unless strategy.nil?)
|
|
131
120
|
end
|
|
132
121
|
|
|
133
122
|
##
|
|
@@ -171,8 +160,8 @@ module Html2rss
|
|
|
171
160
|
# @return [void]
|
|
172
161
|
def initialize_copy(other)
|
|
173
162
|
super
|
|
174
|
-
@headers =
|
|
175
|
-
@stylesheets =
|
|
163
|
+
@headers = other.headers.dup if other.headers.is_a?(Hash)
|
|
164
|
+
@stylesheets = other.stylesheets.map(&:dup)
|
|
176
165
|
end
|
|
177
166
|
|
|
178
167
|
private
|
|
@@ -32,15 +32,13 @@ module Html2rss
|
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
module_function
|
|
36
|
-
|
|
37
35
|
ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
|
|
38
36
|
private_constant :ENV_VAR
|
|
39
37
|
|
|
40
38
|
##
|
|
41
39
|
# @param sample_url [String, nil] optional URL for a sample scrape smoke test
|
|
42
40
|
# @return [Result]
|
|
43
|
-
def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
|
|
41
|
+
def self.call(sample_url: nil) # rubocop:disable Metrics/AbcSize
|
|
44
42
|
env = env_check
|
|
45
43
|
checks = [env_check_record(env)]
|
|
46
44
|
return missing_env_failure(checks) unless checks.first.ok
|
|
@@ -57,7 +55,7 @@ module Html2rss
|
|
|
57
55
|
|
|
58
56
|
##
|
|
59
57
|
# @return [Hash{Symbol => Object}]
|
|
60
|
-
def env_check
|
|
58
|
+
def self.env_check
|
|
61
59
|
base = ENV.fetch(ENV_VAR, '').strip
|
|
62
60
|
{
|
|
63
61
|
configured: MCP::Runtime.botasaurus_configured?,
|
|
@@ -69,7 +67,7 @@ module Html2rss
|
|
|
69
67
|
##
|
|
70
68
|
# @param base_url [String]
|
|
71
69
|
# @return [Hash{Symbol => Object}]
|
|
72
|
-
def health_check(base_url)
|
|
70
|
+
def self.health_check(base_url)
|
|
73
71
|
endpoint = "#{Url.for_channel(base_url).to_s.chomp('/')}/health"
|
|
74
72
|
response = health_session.get(endpoint)
|
|
75
73
|
raise response.error if response.is_a?(HTTPX::ErrorResponse)
|
|
@@ -80,60 +78,54 @@ module Html2rss
|
|
|
80
78
|
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
81
79
|
end
|
|
82
80
|
|
|
83
|
-
def health_session
|
|
81
|
+
def self.health_session
|
|
84
82
|
HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
|
|
85
83
|
resolver_class: :system)
|
|
86
84
|
end
|
|
87
|
-
module_function :health_session
|
|
88
85
|
private_class_method :health_session
|
|
89
86
|
|
|
90
87
|
##
|
|
91
88
|
# @param url [String]
|
|
92
89
|
# @return [Hash{Symbol => Object}]
|
|
93
|
-
def sample_scrape(url)
|
|
90
|
+
def self.sample_scrape(url)
|
|
94
91
|
wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
|
|
95
92
|
{
|
|
96
93
|
ok: wire[:status].to_i.between?(200, 399),
|
|
97
94
|
status: wire[:status],
|
|
98
95
|
articles_count: wire[:articles_count]
|
|
99
96
|
}
|
|
100
|
-
rescue
|
|
97
|
+
rescue Html2rss::Error => error
|
|
101
98
|
{ ok: false, error: "#{error.class}: #{error.message}" }
|
|
102
99
|
end
|
|
103
100
|
|
|
104
|
-
def env_check_record(env = env_check)
|
|
101
|
+
def self.env_check_record(env = env_check)
|
|
105
102
|
Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
|
|
106
103
|
end
|
|
107
|
-
module_function :env_check_record
|
|
108
104
|
private_class_method :env_check_record
|
|
109
105
|
|
|
110
|
-
def health_check_record(base_url)
|
|
106
|
+
def self.health_check_record(base_url)
|
|
111
107
|
return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
|
|
112
108
|
|
|
113
109
|
health = health_check(base_url)
|
|
114
110
|
Check.new(name: :health, ok: health[:ok], detail: health)
|
|
115
111
|
end
|
|
116
|
-
module_function :health_check_record
|
|
117
112
|
private_class_method :health_check_record
|
|
118
113
|
|
|
119
|
-
def sample_scrape_record(url)
|
|
114
|
+
def self.sample_scrape_record(url)
|
|
120
115
|
sample = sample_scrape(url)
|
|
121
116
|
Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
|
|
122
117
|
end
|
|
123
|
-
module_function :sample_scrape_record
|
|
124
118
|
private_class_method :sample_scrape_record
|
|
125
119
|
|
|
126
|
-
def missing_env_failure(checks)
|
|
120
|
+
def self.missing_env_failure(checks)
|
|
127
121
|
Log.info("doctor botasaurus: #{ENV_VAR} unset")
|
|
128
122
|
failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
|
|
129
123
|
end
|
|
130
|
-
module_function :missing_env_failure
|
|
131
124
|
private_class_method :missing_env_failure
|
|
132
125
|
|
|
133
|
-
def failure(checks, message)
|
|
126
|
+
def self.failure(checks, message)
|
|
134
127
|
Result.new(ok: false, checks:, message:)
|
|
135
128
|
end
|
|
136
|
-
module_function :failure
|
|
137
129
|
private_class_method :failure
|
|
138
130
|
end
|
|
139
131
|
end
|