html2rss 0.28.0 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +82 -25
  3. data/html2rss.gemspec +0 -1
  4. data/lib/html2rss/article.rb +1 -2
  5. data/lib/html2rss/auto_source/README.md +8 -8
  6. data/lib/html2rss/auto_source/cleanup.rb +92 -1
  7. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
  8. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
  9. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
  10. data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
  11. data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
  12. data/lib/html2rss/auto_source/scraper.rb +2 -3
  13. data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
  14. data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
  15. data/lib/html2rss/auto_source/segmenter.rb +1 -2
  16. data/lib/html2rss/auto_source.rb +25 -11
  17. data/lib/html2rss/batch.rb +169 -0
  18. data/lib/html2rss/capture/README.md +15 -6
  19. data/lib/html2rss/capture.rb +158 -15
  20. data/lib/html2rss/channel.rb +1 -2
  21. data/lib/html2rss/cli/probe_view.rb +66 -0
  22. data/lib/html2rss/cli/render.rb +124 -0
  23. data/lib/html2rss/cli/validate.rb +91 -0
  24. data/lib/html2rss/cli.rb +318 -193
  25. data/lib/html2rss/config/request_headers.rb +4 -14
  26. data/lib/html2rss/config/schema.rb +7 -5
  27. data/lib/html2rss/config/validator.rb +1 -0
  28. data/lib/html2rss/config.rb +59 -22
  29. data/lib/html2rss/doctor/botasaurus.rb +130 -0
  30. data/lib/html2rss/feed_builder/rss.rb +1 -2
  31. data/lib/html2rss/feed_pipeline/README.md +5 -5
  32. data/lib/html2rss/feed_pipeline/auto_fallback.rb +29 -10
  33. data/lib/html2rss/feed_pipeline.rb +27 -16
  34. data/lib/html2rss/feed_resolution/README.md +11 -11
  35. data/lib/html2rss/feed_resolution.rb +6 -12
  36. data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
  37. data/lib/html2rss/html/article_extractor.rb +4 -9
  38. data/lib/html2rss/html/feed_link.rb +4 -2
  39. data/lib/html2rss/html/navigator.rb +4 -39
  40. data/lib/html2rss/html/probe.rb +82 -0
  41. data/lib/html2rss/html/sst_article_extractor.rb +1 -2
  42. data/lib/html2rss/link_destination/noise_policy.rb +1 -2
  43. data/lib/html2rss/link_destination/path_classifier.rb +20 -19
  44. data/lib/html2rss/mcp/README.md +80 -0
  45. data/lib/html2rss/mcp/contract.rb +135 -13
  46. data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
  47. data/lib/html2rss/mcp/outcome.rb +112 -42
  48. data/lib/html2rss/mcp/runtime.rb +45 -0
  49. data/lib/html2rss/mcp/server/tools.rb +277 -0
  50. data/lib/html2rss/mcp/server.rb +91 -232
  51. data/lib/html2rss/mcp.rb +3 -0
  52. data/lib/html2rss/page_recon/README.md +69 -0
  53. data/lib/html2rss/page_recon/diagnostics.rb +211 -0
  54. data/lib/html2rss/page_recon.rb +56 -2
  55. data/lib/html2rss/recon.rb +252 -0
  56. data/lib/html2rss/request_service/blocked_surface.rb +1 -0
  57. data/lib/html2rss/request_service/faraday_strategy.rb +11 -7
  58. data/lib/html2rss/request_service/policy.rb +1 -2
  59. data/lib/html2rss/request_service/response.rb +3 -4
  60. data/lib/html2rss/request_service/strategy.rb +1 -2
  61. data/lib/html2rss/request_service.rb +2 -0
  62. data/lib/html2rss/request_session.rb +1 -2
  63. data/lib/html2rss/scrape_target.rb +1 -1
  64. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
  65. data/lib/html2rss/selectors/post_processors.rb +0 -1
  66. data/lib/html2rss/selectors.rb +33 -10
  67. data/lib/html2rss/sst/attrs.rb +1 -2
  68. data/lib/html2rss/sst/normalizer.rb +49 -11
  69. data/lib/html2rss/status.rb +3 -6
  70. data/lib/html2rss/syndication/README.md +6 -6
  71. data/lib/html2rss/syndication/discovery.rb +2 -4
  72. data/lib/html2rss/test/enhance_audit.rb +192 -0
  73. data/lib/html2rss/test.rb +458 -0
  74. data/lib/html2rss/url.rb +1 -2
  75. data/lib/html2rss/version.rb +1 -1
  76. data/lib/html2rss.rb +157 -35
  77. data/schema/html2rss-config.schema.json +9 -29
  78. metadata +17 -18
  79. data/lib/html2rss/mcp/inspect.rb +0 -138
  80. data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Html2rss
6
+ class CLI
7
+ ##
8
+ # Text and JSON rendering for CLI command output (display only; no policy).
9
+ module Render
10
+ # ANSI colors for recon verdict labels in text output.
11
+ VERDICT_COLORS = { build: "\e[32m", defer: "\e[33m" }.freeze
12
+
13
+ module_function
14
+
15
+ ##
16
+ # @param results [Array<Hash>]
17
+ # @param format [String]
18
+ # @param batch_mode [Boolean]
19
+ # @return [void]
20
+ def inspect_output(results, format:, batch_mode:)
21
+ if format == 'json'
22
+ json(results, batch_mode)
23
+ else
24
+ results.each { |data| probe_card(ProbeView.from_wire(data)) }
25
+ end
26
+ end
27
+
28
+ ##
29
+ # @param results [Array<Html2rss::Recon::Result>]
30
+ # @param format [String]
31
+ # @param batch_mode [Boolean]
32
+ # @param url_only [Boolean]
33
+ # @return [void]
34
+ def recon_output(results, format:, batch_mode:, url_only: false) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
35
+ if url_only
36
+ results.each { |r| puts r.requested_url }
37
+ elsif format == 'json'
38
+ json(results.map(&:to_h), batch_mode)
39
+ elsif format == 'tsv'
40
+ puts %w[verdict status requested_url final_url native_feed notes].join("\t")
41
+ results.each do |r|
42
+ row = [
43
+ r.verdict.to_s.upcase,
44
+ r.status || '-',
45
+ r.requested_url,
46
+ r.final_url,
47
+ r.native_feed || '-',
48
+ r.notes.join('; ')
49
+ ]
50
+ puts row.join("\t")
51
+ end
52
+ else
53
+ results.each { |r| probe_card(ProbeView.from_recon(r)) }
54
+ end
55
+ end
56
+
57
+ ##
58
+ # @param view [ProbeView]
59
+ # @return [void]
60
+ def probe_card(view) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
61
+ if view.verdict
62
+ color = VERDICT_COLORS.fetch(view.verdict.to_sym, "\e[31m")
63
+ puts "#{color}[#{view.verdict.to_s.upcase}]\e[0m #{view.requested}"
64
+ else
65
+ puts view.requested
66
+ end
67
+ probe_lines(view)
68
+ if view.alternate_feeds.any?
69
+ hrefs = view.alternate_feeds.filter_map { |f| ProbeView.wire_val(f, :href) }
70
+ puts " Feeds: #{hrefs.join(', ')}" if hrefs.any?
71
+ end
72
+ puts " Feed: #{view.native_feed}" if view.native_feed
73
+ puts " Notes: #{view.notes.join(', ')}" if view.notes.any?
74
+ puts " Strategy: #{view.strategy}" if view.strategy
75
+ puts ''
76
+ end
77
+
78
+ ##
79
+ # @param result [Html2rss::Test::Result]
80
+ # @param source [String]
81
+ # @return [void]
82
+ def test_card(result, source) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
83
+ if result.success
84
+ puts "\e[32m✓ Schema valid\e[0m (#{source})"
85
+ dur = "#{result.duration_seconds}s"
86
+ puts "\e[32m✓ Extracted #{result.item_count} items in #{dur}\e[0m (strategy: #{result.strategy_used})"
87
+ puts "\nChannel: #{result.channel_title} (#{result.channel_url})"
88
+ if result.sample_items.any?
89
+ puts 'Sample items:'
90
+ result.sample_items.each_with_index do |item, i|
91
+ puts " #{i + 1}. #{"#{item[:published_at]} | " if item[:published_at]}#{item[:title]}"
92
+ puts " #{item[:url]}"
93
+ end
94
+ end
95
+ else
96
+ warn "\e[31m✗ Test failed\e[0m (#{source})"
97
+ warn " Error: #{result.error_message}" if result.error_message
98
+ result.validation_errors&.each { |k, v| warn " Schema error [#{k}]: #{Array(v).join(', ')}" }
99
+ end
100
+ end
101
+
102
+ ##
103
+ # @param data [Object]
104
+ # @param batch_mode [Boolean]
105
+ # @return [void]
106
+ def json(data, batch_mode)
107
+ payload = batch_mode ? data : data.first
108
+ puts JSON.pretty_generate(payload)
109
+ end
110
+
111
+ ##
112
+ # @param view [ProbeView]
113
+ # @return [void]
114
+ def probe_lines(view)
115
+ if view.final && view.final != view.requested
116
+ puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
117
+ end
118
+ surface = view.surface.respond_to?(:to_s) ? view.surface.to_s : view.surface
119
+ puts " Surface: #{surface} (#{view.articles_count} articles)"
120
+ end
121
+ private_class_method :probe_lines
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ class CLI
5
+ ##
6
+ # Validate command: file/glob/stdin branching and multi-file reporting.
7
+ module Validate
8
+ module_function
9
+
10
+ ##
11
+ # @param files [Array<String>]
12
+ # @param params [Hash]
13
+ # @param quiet [Boolean]
14
+ # @return [void]
15
+ # @raise [Thor::Error]
16
+ def run(files, params:, quiet:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
17
+ if files.size == 2 && File.file?(files[0].to_s) && !File.exist?(files[1].to_s)
18
+ raise Thor::Error, "No such file: #{files[1]}" if path_like_config?(files[1])
19
+
20
+ return run_named_feed(files[0], files[1], params:, quiet:)
21
+ end
22
+
23
+ target_files = resolve_files(files)
24
+ failed = []
25
+
26
+ target_files.each do |file|
27
+ result = validate_file(file, params:)
28
+
29
+ if result.success?
30
+ puts(target_files.size == 1 ? 'Configuration is valid' : "ok #{file}") unless quiet
31
+ else
32
+ error_details = result.errors.to_h
33
+ raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
34
+
35
+ warn "FAIL #{file}"
36
+ error_details.each { |key, msg| warn " #{key}: #{Array(msg).join(', ')}" }
37
+ failed << file
38
+ end
39
+ end
40
+
41
+ return if failed.empty?
42
+
43
+ raise Thor::Error, "#{failed.size}/#{target_files.size} configurations failed validation."
44
+ end
45
+
46
+ ##
47
+ # @param files [Array<String>]
48
+ # @return [Array<String>]
49
+ def resolve_files(files)
50
+ return ['-'] if files.empty? || files == ['-']
51
+
52
+ resolved = []
53
+ files.each do |f|
54
+ matched = Dir.glob(f)
55
+ resolved.concat(matched.empty? ? [f] : matched)
56
+ end
57
+ resolved.uniq
58
+ end
59
+
60
+ def run_named_feed(file, feed_name, params:, quiet:)
61
+ result = Html2rss.validate(file, feed_name, params:)
62
+ raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
63
+
64
+ puts 'Configuration is valid' unless quiet
65
+ end
66
+ module_function :run_named_feed
67
+ private_class_method :run_named_feed
68
+
69
+ def validate_file(file, params:)
70
+ if file == '-'
71
+ Html2rss.validate($stdin.read, params:)
72
+ else
73
+ Html2rss.validate(file, params:)
74
+ end
75
+ end
76
+ module_function :validate_file
77
+ private_class_method :validate_file
78
+
79
+ def path_like_config?(arg)
80
+ path = arg.to_s
81
+ return true if path.start_with?('-')
82
+ return true if path.end_with?('.yml', '.yaml')
83
+ return true if path.include?('/') || path.include?('*')
84
+
85
+ false
86
+ end
87
+ module_function :path_like_config?
88
+ private_class_method :path_like_config?
89
+ end
90
+ end
91
+ end