html2rss 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +82 -25
- data/html2rss.gemspec +0 -1
- data/lib/html2rss/article.rb +1 -2
- data/lib/html2rss/auto_source/README.md +8 -8
- data/lib/html2rss/auto_source/cleanup.rb +92 -1
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
- data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
- data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
- data/lib/html2rss/auto_source/scraper.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
- data/lib/html2rss/auto_source/segmenter.rb +1 -2
- data/lib/html2rss/auto_source.rb +25 -11
- data/lib/html2rss/batch.rb +169 -0
- data/lib/html2rss/capture/README.md +15 -6
- data/lib/html2rss/capture.rb +158 -15
- data/lib/html2rss/channel.rb +1 -2
- data/lib/html2rss/cli/probe_view.rb +66 -0
- data/lib/html2rss/cli/render.rb +124 -0
- data/lib/html2rss/cli/validate.rb +91 -0
- data/lib/html2rss/cli.rb +318 -193
- data/lib/html2rss/config/request_headers.rb +4 -14
- data/lib/html2rss/config/schema.rb +7 -5
- data/lib/html2rss/config/validator.rb +1 -0
- data/lib/html2rss/config.rb +59 -22
- data/lib/html2rss/doctor/botasaurus.rb +130 -0
- data/lib/html2rss/feed_builder/rss.rb +1 -2
- data/lib/html2rss/feed_pipeline/README.md +5 -5
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
- data/lib/html2rss/feed_pipeline.rb +27 -16
- data/lib/html2rss/feed_resolution/README.md +11 -11
- data/lib/html2rss/feed_resolution.rb +6 -12
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
- data/lib/html2rss/html/article_extractor.rb +4 -9
- data/lib/html2rss/html/feed_link.rb +4 -2
- data/lib/html2rss/html/navigator.rb +4 -39
- data/lib/html2rss/html/probe.rb +82 -0
- data/lib/html2rss/html/sst_article_extractor.rb +1 -2
- data/lib/html2rss/link_destination/noise_policy.rb +1 -2
- data/lib/html2rss/link_destination/path_classifier.rb +20 -19
- data/lib/html2rss/mcp/README.md +80 -0
- data/lib/html2rss/mcp/contract.rb +135 -13
- data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
- data/lib/html2rss/mcp/outcome.rb +112 -42
- data/lib/html2rss/mcp/runtime.rb +45 -0
- data/lib/html2rss/mcp/server/tools.rb +277 -0
- data/lib/html2rss/mcp/server.rb +91 -232
- data/lib/html2rss/mcp.rb +3 -0
- data/lib/html2rss/page_recon/README.md +69 -0
- data/lib/html2rss/page_recon/diagnostics.rb +211 -0
- data/lib/html2rss/page_recon.rb +56 -2
- data/lib/html2rss/recon.rb +252 -0
- data/lib/html2rss/request_service/blocked_surface.rb +1 -0
- data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
- data/lib/html2rss/request_service/policy.rb +1 -2
- data/lib/html2rss/request_service/response.rb +3 -4
- data/lib/html2rss/request_service/strategy.rb +1 -2
- data/lib/html2rss/request_service.rb +2 -0
- data/lib/html2rss/request_session.rb +1 -2
- data/lib/html2rss/scrape_target.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
- data/lib/html2rss/selectors/post_processors.rb +0 -1
- data/lib/html2rss/selectors.rb +33 -10
- data/lib/html2rss/sst/attrs.rb +1 -2
- data/lib/html2rss/sst/normalizer.rb +49 -11
- data/lib/html2rss/status.rb +3 -6
- data/lib/html2rss/syndication/README.md +6 -6
- data/lib/html2rss/syndication/discovery.rb +2 -4
- data/lib/html2rss/test/enhance_audit.rb +192 -0
- data/lib/html2rss/test.rb +458 -0
- data/lib/html2rss/url.rb +1 -2
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +157 -35
- data/schema/html2rss-config.schema.json +9 -29
- metadata +17 -18
- data/lib/html2rss/mcp/inspect.rb +0 -138
- data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Html2rss
|
|
6
|
+
class CLI
|
|
7
|
+
##
|
|
8
|
+
# Text and JSON rendering for CLI command output (display only; no policy).
|
|
9
|
+
module Render
|
|
10
|
+
# ANSI colors for recon verdict labels in text output.
|
|
11
|
+
VERDICT_COLORS = { build: "\e[32m", defer: "\e[33m" }.freeze
|
|
12
|
+
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
##
|
|
16
|
+
# @param results [Array<Hash>]
|
|
17
|
+
# @param format [String]
|
|
18
|
+
# @param batch_mode [Boolean]
|
|
19
|
+
# @return [void]
|
|
20
|
+
def inspect_output(results, format:, batch_mode:)
|
|
21
|
+
if format == 'json'
|
|
22
|
+
json(results, batch_mode)
|
|
23
|
+
else
|
|
24
|
+
results.each { |data| probe_card(ProbeView.from_wire(data)) }
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
##
|
|
29
|
+
# @param results [Array<Html2rss::Recon::Result>]
|
|
30
|
+
# @param format [String]
|
|
31
|
+
# @param batch_mode [Boolean]
|
|
32
|
+
# @param url_only [Boolean]
|
|
33
|
+
# @return [void]
|
|
34
|
+
def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
35
|
+
if url_only
|
|
36
|
+
results.each { |r| puts r.requested_url }
|
|
37
|
+
elsif format == 'json'
|
|
38
|
+
json(results.map(&:to_h), batch_mode)
|
|
39
|
+
elsif format == 'tsv'
|
|
40
|
+
puts %w[verdict status requested_url final_url native_feed notes].join("\t")
|
|
41
|
+
results.each do |r|
|
|
42
|
+
row = [
|
|
43
|
+
r.verdict.to_s.upcase,
|
|
44
|
+
r.status || '-',
|
|
45
|
+
r.requested_url,
|
|
46
|
+
r.final_url,
|
|
47
|
+
r.native_feed || '-',
|
|
48
|
+
r.notes.join('; ')
|
|
49
|
+
]
|
|
50
|
+
puts row.join("\t")
|
|
51
|
+
end
|
|
52
|
+
else
|
|
53
|
+
results.each { |r| probe_card(ProbeView.from_recon(r)) }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
##
|
|
58
|
+
# @param view [ProbeView]
|
|
59
|
+
# @return [void]
|
|
60
|
+
def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
61
|
+
if view.verdict
|
|
62
|
+
color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
|
|
63
|
+
puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
|
|
64
|
+
else
|
|
65
|
+
puts view.requested
|
|
66
|
+
end
|
|
67
|
+
probe_lines(view)
|
|
68
|
+
if view.alternate_feeds.any?
|
|
69
|
+
hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
|
|
70
|
+
puts " Feeds: #{hrefs.join(', ')}" if hrefs.any?
|
|
71
|
+
end
|
|
72
|
+
puts " Feed: #{view.native_feed}" if view.native_feed
|
|
73
|
+
puts " Notes: #{view.notes.join(', ')}" if view.notes.any?
|
|
74
|
+
puts " Strategy: #{view.strategy}" if view.strategy
|
|
75
|
+
puts ''
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
##
|
|
79
|
+
# @param result [Html2rss::Test::Result]
|
|
80
|
+
# @param source [String]
|
|
81
|
+
# @return [void]
|
|
82
|
+
def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
83
|
+
if result.success
|
|
84
|
+
puts "\e[32m✓ Schema valid\e[0m (#{source})"
|
|
85
|
+
dur = "#{result.duration_seconds}s"
|
|
86
|
+
puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
|
|
87
|
+
puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
|
|
88
|
+
if result.sample_items.any?
|
|
89
|
+
puts 'Sample items:'
|
|
90
|
+
result.sample_items.each_with_index do |item, i|
|
|
91
|
+
puts " #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
|
|
92
|
+
puts " #{item[:url]}"
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
else
|
|
96
|
+
warn "\e[31m✗ Test failed\e[0m (#{source})"
|
|
97
|
+
warn " Error: #{result.error_message}" if result.error_message
|
|
98
|
+
result.validation_errors&.each { |k, v| warn " Schema error [#{k}]: #{Array(v).join(', ')}" }
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
##
|
|
103
|
+
# @param data [Object]
|
|
104
|
+
# @param batch_mode [Boolean]
|
|
105
|
+
# @return [void]
|
|
106
|
+
def json(data, batch_mode)
|
|
107
|
+
payload = batch_mode ? data : data.first
|
|
108
|
+
puts JSON.pretty_generate(payload)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
##
|
|
112
|
+
# @param view [ProbeView]
|
|
113
|
+
# @return [void]
|
|
114
|
+
def probe_lines(view)
|
|
115
|
+
if view.final && view.final != view.requested
|
|
116
|
+
puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
|
|
117
|
+
end
|
|
118
|
+
surface = view.surface.respond_to?(:to_s) ? view.surface.to_s : view.surface
|
|
119
|
+
puts " Surface: #{surface} (#{view.articles_count} articles)"
|
|
120
|
+
end
|
|
121
|
+
private_class_method :probe_lines
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class CLI
|
|
5
|
+
##
|
|
6
|
+
# Validate command: file/glob/stdin branching and multi-file reporting.
|
|
7
|
+
module Validate
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
##
|
|
11
|
+
# @param files [Array<String>]
|
|
12
|
+
# @param params [Hash]
|
|
13
|
+
# @param quiet [Boolean]
|
|
14
|
+
# @return [void]
|
|
15
|
+
# @raise [Thor::Error]
|
|
16
|
+
def run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
17
|
+
if files.size == 2 && File.file?(files[0].to_s) && !File.exist?(files[1].to_s)
|
|
18
|
+
raise Thor::Error, "No such file: #{files[1]}" if path_like_config?(files[1])
|
|
19
|
+
|
|
20
|
+
return run_named_feed(files[0], files[1], params:, quiet:)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
target_files = resolve_files(files)
|
|
24
|
+
failed = []
|
|
25
|
+
|
|
26
|
+
target_files.each do |file|
|
|
27
|
+
result = validate_file(file, params:)
|
|
28
|
+
|
|
29
|
+
if result.success?
|
|
30
|
+
puts(target_files.size == 1 ? 'Configuration is valid' : "ok #{file}") unless quiet
|
|
31
|
+
else
|
|
32
|
+
error_details = result.errors.to_h
|
|
33
|
+
raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
|
|
34
|
+
|
|
35
|
+
warn "FAIL #{file}"
|
|
36
|
+
error_details.each { |key, msg| warn " #{key}: #{Array(msg).join(', ')}" }
|
|
37
|
+
failed << file
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
return if failed.empty?
|
|
42
|
+
|
|
43
|
+
raise Thor::Error, "#{failed.size}/#{target_files.size} configurations failed validation."
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
##
|
|
47
|
+
# @param files [Array<String>]
|
|
48
|
+
# @return [Array<String>]
|
|
49
|
+
def resolve_files(files)
|
|
50
|
+
return ['-'] if files.empty? || files == ['-']
|
|
51
|
+
|
|
52
|
+
resolved = []
|
|
53
|
+
files.each do |f|
|
|
54
|
+
matched = Dir.glob(f)
|
|
55
|
+
resolved.concat(matched.empty? ? [f] : matched)
|
|
56
|
+
end
|
|
57
|
+
resolved.uniq
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def run_named_feed(file, feed_name, params:, quiet:)
|
|
61
|
+
result = Html2rss.validate(file, feed_name, params:)
|
|
62
|
+
raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
|
|
63
|
+
|
|
64
|
+
puts 'Configuration is valid' unless quiet
|
|
65
|
+
end
|
|
66
|
+
module_function :run_named_feed
|
|
67
|
+
private_class_method :run_named_feed
|
|
68
|
+
|
|
69
|
+
def validate_file(file, params:)
|
|
70
|
+
if file == '-'
|
|
71
|
+
Html2rss.validate($stdin.read, params:)
|
|
72
|
+
else
|
|
73
|
+
Html2rss.validate(file, params:)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
module_function :validate_file
|
|
77
|
+
private_class_method :validate_file
|
|
78
|
+
|
|
79
|
+
def path_like_config?(arg)
|
|
80
|
+
path = arg.to_s
|
|
81
|
+
return true if path.start_with?('-')
|
|
82
|
+
return true if path.end_with?('.yml', '.yaml')
|
|
83
|
+
return true if path.include?('/') || path.include?('*')
|
|
84
|
+
|
|
85
|
+
false
|
|
86
|
+
end
|
|
87
|
+
module_function :path_like_config?
|
|
88
|
+
private_class_method :path_like_config?
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|