html2rss 0.24.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. checksums.yaml +4 -4
  2. data/lib/html2rss/auto_source/scraper/html.rb +68 -145
  3. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +116 -0
  4. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +63 -0
  5. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +179 -0
  6. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +55 -0
  7. data/lib/html2rss/auto_source/scraper/json_state.rb +0 -379
  8. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +35 -30
  9. data/lib/html2rss/auto_source/scraper/semantic_html.rb +40 -120
  10. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +161 -0
  11. data/lib/html2rss/auto_source/scraper/sitemap.rb +10 -7
  12. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +6 -2
  13. data/lib/html2rss/auto_source/scraper.rb +81 -44
  14. data/lib/html2rss/auto_source/segment.rb +29 -0
  15. data/lib/html2rss/auto_source/segmenter/cluster.rb +163 -0
  16. data/lib/html2rss/auto_source/segmenter/list.rb +101 -0
  17. data/lib/html2rss/auto_source/segmenter/primary_link.rb +81 -0
  18. data/lib/html2rss/auto_source/segmenter/semantic.rb +104 -0
  19. data/lib/html2rss/auto_source/segmenter.rb +66 -0
  20. data/lib/html2rss/auto_source.rb +74 -21
  21. data/lib/html2rss/cli.rb +8 -13
  22. data/lib/html2rss/config/auto_source_contract.rb +2 -0
  23. data/lib/html2rss/config/request_controls.rb +33 -0
  24. data/lib/html2rss/config/validator.rb +15 -8
  25. data/lib/html2rss/config.rb +6 -2
  26. data/lib/html2rss/feed_builder/rss.rb +17 -5
  27. data/lib/html2rss/feed_pipeline.rb +4 -0
  28. data/lib/html2rss/feed_result.rb +1 -1
  29. data/lib/html2rss/html/article_extractor/category_extractor.rb +10 -36
  30. data/lib/html2rss/html/article_extractor/date_extractor.rb +2 -7
  31. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +5 -50
  32. data/lib/html2rss/html/article_extractor/image_extractor.rb +5 -16
  33. data/lib/html2rss/html/article_rules/category.rb +55 -0
  34. data/lib/html2rss/html/article_rules/date.rb +25 -0
  35. data/lib/html2rss/html/article_rules/enclosure.rb +72 -0
  36. data/lib/html2rss/html/article_rules/image.rb +109 -0
  37. data/lib/html2rss/html/article_rules.rb +10 -0
  38. data/lib/html2rss/html/rendering/audio_renderer.rb +2 -12
  39. data/lib/html2rss/html/rendering/escaped_attributes.rb +27 -0
  40. data/lib/html2rss/html/rendering/image_renderer.rb +2 -12
  41. data/lib/html2rss/html/rendering/pdf_renderer.rb +2 -8
  42. data/lib/html2rss/html/rendering/video_renderer.rb +2 -12
  43. data/lib/html2rss/html/sst_article_extractor.rb +279 -0
  44. data/lib/html2rss/link_destination/destination_facts.rb +40 -0
  45. data/lib/html2rss/link_destination/noise_policy.rb +79 -0
  46. data/lib/html2rss/link_destination/path_classifier.rb +205 -0
  47. data/lib/html2rss/link_destination/text_classifier.rb +64 -0
  48. data/lib/html2rss/link_destination.rb +8 -0
  49. data/lib/html2rss/request_service/faraday_strategy.rb +42 -1
  50. data/lib/html2rss/request_service/network_guard.rb +5 -3
  51. data/lib/html2rss/scoring/anchor_score.rb +34 -0
  52. data/lib/html2rss/scoring/cluster_scorer.rb +87 -0
  53. data/lib/html2rss/scoring/container_assessor.rb +83 -0
  54. data/lib/html2rss/scoring/engine.rb +101 -0
  55. data/lib/html2rss/scoring/link_resolver.rb +72 -0
  56. data/lib/html2rss/scoring/observation.rb +53 -0
  57. data/lib/html2rss/scoring/ranked_segment.rb +45 -0
  58. data/lib/html2rss/scoring/score.rb +30 -0
  59. data/lib/html2rss/scoring.rb +33 -0
  60. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +10 -1
  61. data/lib/html2rss/selectors.rb +0 -20
  62. data/lib/html2rss/sst/attrs.rb +91 -0
  63. data/lib/html2rss/sst/document.rb +23 -0
  64. data/lib/html2rss/sst/index.rb +112 -0
  65. data/lib/html2rss/sst/node.rb +147 -0
  66. data/lib/html2rss/sst/normalizer.rb +171 -0
  67. data/lib/html2rss/sst/tags.rb +26 -0
  68. data/lib/html2rss/sst/text.rb +81 -0
  69. data/lib/html2rss/sst.rb +8 -0
  70. data/lib/html2rss/version.rb +1 -1
  71. data/lib/html2rss.rb +26 -26
  72. data/schema/html2rss-config.schema.json +20 -0
  73. metadata +42 -18
  74. data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +0 -80
  75. data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +0 -86
  76. data/lib/html2rss/auto_source/discovery/dom_clustering.rb +0 -119
  77. data/lib/html2rss/auto_source/discovery/list_candidates.rb +0 -94
  78. data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +0 -219
  79. data/lib/html2rss/auto_source/discovery/semantic_containers.rb +0 -71
  80. data/lib/html2rss/auto_source/discovery/sitemap.rb +0 -159
  81. data/lib/html2rss/auto_source/discovery.rb +0 -14
  82. data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +0 -28
  83. data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +0 -106
  84. data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +0 -80
  85. data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +0 -42
  86. data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +0 -38
  87. data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +0 -221
  88. data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +0 -66
  89. data/lib/html2rss/auto_source/link_heuristics.rb +0 -139
@@ -4,31 +4,12 @@ module Html2rss
4
4
  class AutoSource
5
5
  module Scraper
6
6
  ##
7
- # Scrapes semantic containers by choosing one primary content link per
8
- # block before extraction.
9
- #
10
- # This scraper is intentionally container-first:
11
- # 1. collect candidate semantic containers once
12
- # 2. select the strongest content-like anchor within each container
13
- # 3. extract fields from the container while honoring that anchor choice
14
- #
15
- # The result is lower recall on weak-signal blocks, but much better link
16
- # quality on modern teaser cards that mix headlines, utility links, and
17
- # duplicate image overlays.
7
+ # Scrapes semantic containers via SST: Normalizer Segmenter Scoring → SstArticleExtractor.
18
8
  class SemanticHtml
19
9
  include Enumerable
20
10
 
21
- # Container plus selected anchor, scoring metadata, and extracted article.
22
- Entry = Data.define(
23
- :container,
24
- :selected_anchor,
25
- :destination_facts,
26
- :quality_score,
27
- :junk_score,
28
- :final_score,
29
- :position,
30
- :article
31
- )
11
+ # Maximum articles materialized after ranking.
12
+ TOP_K = Scoring::Engine::TOP_K
32
13
 
33
14
  ##
34
15
  # @return [Symbol] config key used to enable or configure this scraper
@@ -40,133 +21,72 @@ module Html2rss
40
21
  return false unless parsed_body
41
22
 
42
23
  new(parsed_body, url: 'https://example.com').extractable?
24
+ rescue ArgumentError
25
+ false
43
26
  end
44
27
 
45
- # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
28
+ # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
46
29
  # @param url [String, Html2rss::Url] base url
47
- # @param extractor [Class] extractor class used for article extraction
30
+ # @param document [SST::Document, nil] memoized SST document from AutoSource
31
+ # @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
48
32
  # @param opts [Hash] scraper-specific options
49
33
  # @option opts [Boolean] :fallback_anchorless whether to keep containers without a primary anchor
50
- def initialize(parsed_body, url:, extractor: Html2rss::Html::ArticleExtractor, **opts)
34
+ def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
51
35
  @parsed_body = parsed_body
36
+ @provided_document = document
37
+ @provided_link_resolver = link_resolver
52
38
  @url = url
53
- @extractor = extractor
54
39
  @permit_unanchored = opts.fetch(:fallback_anchorless, false)
55
- @link_heuristics = LinkHeuristics.new(url)
56
40
  end
57
41
 
58
- attr_reader :parsed_body
59
-
60
42
  ##
61
- # Yields extracted article hashes for each semantic container that
62
- # survives anchor selection.
63
- #
64
- # Detection and extraction share the same memoized entry list so this
65
- # scraper does not rerun anchor ranking once a page has already been
66
- # accepted as extractable.
67
- #
68
- # @yieldparam article_hash [Hash] extracted article hash
69
- # @return [Enumerator<Hash>]
43
+ # @yieldparam article [Html2rss::Article]
44
+ # @return [Enumerator<Html2rss::Article>]
70
45
  def each
71
46
  return enum_for(:each) unless block_given?
72
47
 
73
- ranked_entries.each { yield _1.article }
48
+ articles.each { yield _1 }
74
49
  end
75
50
 
76
51
  ##
77
- # Reports whether the page contains at least one semantic container with
78
- # a selectable primary anchor.
79
- #
80
- # @return [Boolean] true when at least one candidate container yields a primary anchor
52
+ # @return [Boolean]
81
53
  def extractable?
82
- extractable_entries.any?
83
- end
84
-
85
- # @return [Array<Nokogiri::XML::Node>]
86
- def candidate_containers
87
- @candidate_containers ||= collect_candidate_containers
88
- end
89
-
90
- # @param container [Nokogiri::XML::Node]
91
- # @return [Nokogiri::XML::Node, nil]
92
- def primary_anchor_for(container)
93
- Discovery::SemanticAnchorCandidates.new(
94
- container,
95
- link_heuristics: @link_heuristics
96
- ).to_a.max_by(&:score)&.anchor
54
+ ranked_segments.any?
55
+ rescue ArgumentError
56
+ false
97
57
  end
98
58
 
99
- # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
100
- def extractable_entries
101
- @extractable_entries ||= candidate_containers.filter_map do |container|
102
- selected_anchor = primary_anchor_for(container)
103
-
104
- next unless selected_anchor || @permit_unanchored
105
-
106
- destination_facts = selected_anchor ? normalized_destination(selected_anchor) : nil
107
- next if selected_anchor && !destination_facts
108
- # Cheap path-only reject before title/DOM hard-junk observations.
109
- next if destination_facts&.high_confidence_junk_path
110
-
111
- signals = @link_heuristics.assess_container(container, selected_anchor, destination_facts:)
112
- next if signals.hard_junk?
59
+ private
113
60
 
114
- Entry.new(
115
- container:,
116
- selected_anchor:,
117
- destination_facts:,
118
- quality_score: signals.quality_score,
119
- junk_score: signals.junk_score,
120
- final_score: signals.final_score,
121
- position: document_position(container),
122
- article: nil
123
- )
61
+ def articles
62
+ @articles ||= begin
63
+ deduplicator = EntryDeduplicator.new(@url, scraper: self.class, link_resolver:)
64
+ entries = deduplicator.call(ranked_segments)
65
+ entries.filter_map { |ranked| deduplicator.article_for(ranked) }
124
66
  end
67
+ rescue ArgumentError
68
+ []
125
69
  end
126
- # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
127
-
128
- # rubocop:disable Metrics/MethodLength
129
- def ranked_entries
130
- @ranked_entries ||= begin
131
- deduplicator = EntryDeduplicator.new(@url, @extractor)
132
- entries = deduplicator.call(extractable_entries)
133
- entries = stable_rank(entries)
134
-
135
- entries.filter_map do |entry|
136
- article = deduplicator.article_for(entry)
137
- next unless article
138
70
 
139
- Entry.new(
140
- container: entry.container,
141
- selected_anchor: entry.selected_anchor,
142
- destination_facts: entry.destination_facts,
143
- quality_score: entry.quality_score,
144
- junk_score: entry.junk_score,
145
- final_score: entry.final_score,
146
- position: entry.position,
147
- article:
148
- )
149
- end
71
+ def ranked_segments
72
+ @ranked_segments ||= begin
73
+ segments = Segmenter.call(
74
+ document,
75
+ base_url: @url,
76
+ strategy: :semantic,
77
+ permit_unanchored: @permit_unanchored,
78
+ link_resolver:
79
+ )
80
+ Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
150
81
  end
151
82
  end
152
- # rubocop:enable Metrics/MethodLength
153
-
154
- def collect_candidate_containers
155
- Discovery::SemanticContainers.call(parsed_body)
156
- end
157
-
158
- private
159
-
160
- def document_position(container)
161
- (@document_positions ||= candidate_containers.each_with_index.to_h).fetch(container)
162
- end
163
83
 
164
- def normalized_destination(anchor)
165
- (@normalized_destinations ||= {}.compare_by_identity)[anchor] ||= @link_heuristics.destination_facts(anchor)
84
+ def link_resolver
85
+ @link_resolver ||= @provided_link_resolver || Scoring::LinkResolver.new(@url)
166
86
  end
167
87
 
168
- def stable_rank(entries)
169
- entries.sort_by { |entry| [-entry.final_score, entry.position] }
88
+ def document
89
+ @document ||= @provided_document || SST::Normalizer.call(@parsed_body)
170
90
  end
171
91
  end
172
92
  end
@@ -0,0 +1,161 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'date'
4
+ require 'nokogiri'
5
+
6
+ module Html2rss
7
+ class AutoSource
8
+ module Scraper
9
+ class Sitemap
10
+ ##
11
+ # Parses XML sitemap documents and extracts prioritized, recent article entries.
12
+ class Parser
13
+ # Default minimum priority threshold (0.0 to 1.0)
14
+ DEFAULT_MIN_PRIORITY = 0.3
15
+ # Default maximum age in days for sitemap entries
16
+ DEFAULT_MAX_AGE_DAYS = 30
17
+ # Google News sitemap XML namespace
18
+ NEWS_NS = 'http://www.google.com/schemas/sitemap-news/0.9'
19
+
20
+ # Result value object returned by .call
21
+ # @!attribute entries [Array<SitemapEntry>] parsed entries (empty for sitemapindex docs)
22
+ # @!attribute sub_sitemap_urls [Array<String>] child sitemap URLs (empty for urlset docs)
23
+ Result = Data.define(:entries, :sub_sitemap_urls)
24
+
25
+ # Immutable facts for one sitemap entry
26
+ SitemapEntry = Data.define(:url, :title, :published_at, :priority, :changefreq)
27
+
28
+ class << self
29
+ ##
30
+ # Parses sitemap XML body and returns a Result.
31
+ #
32
+ # @param xml_content [String] sitemap XML body string
33
+ # @param min_priority [Float] minimum priority score (0.0..1.0)
34
+ # @param max_age_days [Integer] maximum age in days for entries
35
+ # @return [Result] result with entries and/or sub_sitemap_urls
36
+ def call(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
37
+ new(xml_content, min_priority:, max_age_days:).call
38
+ end
39
+ end
40
+
41
+ ##
42
+ # @param xml_content [String]
43
+ # @param min_priority [Float]
44
+ # @param max_age_days [Integer]
45
+ def initialize(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
46
+ @xml_content = xml_content
47
+ @min_priority = min_priority
48
+ @max_age_days = max_age_days
49
+ end
50
+
51
+ ##
52
+ # @return [Result]
53
+ def call
54
+ document = Nokogiri::XML(xml_content)
55
+ return Result.new(entries: [], sub_sitemap_urls: []) if invalid_xml?(document)
56
+
57
+ if sitemapindex?(document)
58
+ Result.new(entries: [], sub_sitemap_urls: extract_sub_sitemap_urls(document))
59
+ else
60
+ Result.new(entries: extract_entries(document), sub_sitemap_urls: [])
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ attr_reader :xml_content, :min_priority, :max_age_days
67
+
68
+ def invalid_xml?(document)
69
+ document.errors.any? && document.at_css('urlset, sitemapindex').nil?
70
+ end
71
+
72
+ def sitemapindex?(document)
73
+ !document.at_xpath('//*[local-name()="sitemapindex"]').nil?
74
+ end
75
+
76
+ def extract_sub_sitemap_urls(document)
77
+ document.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]').filter_map do |node|
78
+ text = node.text.strip
79
+ text unless text.empty?
80
+ end
81
+ end
82
+
83
+ def extract_entries(document)
84
+ document.xpath('//*[local-name()="url"]').filter_map { build_entry(_1) }
85
+ end
86
+
87
+ def build_entry(url_node)
88
+ loc = parse_loc(url_node)
89
+ return unless loc
90
+ return if invalid_entry_signals?(url_node)
91
+
92
+ published_at = parse_date(url_node)
93
+ return if stale_date?(published_at)
94
+
95
+ build_entry_facts(url_node, loc, published_at)
96
+ end
97
+
98
+ def build_entry_facts(url_node, loc, published_at)
99
+ SitemapEntry.new(
100
+ url: loc,
101
+ title: parse_title(url_node),
102
+ published_at:,
103
+ priority: parse_priority(url_node),
104
+ changefreq: parse_changefreq(url_node)
105
+ )
106
+ end
107
+
108
+ def parse_loc(url_node)
109
+ node = url_node.at_xpath('./*[local-name()="loc"]')
110
+ text = node&.text&.strip
111
+ text unless text.to_s.empty?
112
+ end
113
+
114
+ def parse_changefreq(url_node)
115
+ node = url_node.at_xpath('./*[local-name()="changefreq"]')
116
+ node ? node.text.strip.downcase : nil
117
+ end
118
+
119
+ def invalid_entry_signals?(url_node)
120
+ parse_priority(url_node) < min_priority || parse_changefreq(url_node) == 'never'
121
+ end
122
+
123
+ def parse_priority(url_node)
124
+ node = url_node.at_xpath('./*[local-name()="priority"]')
125
+ text = node&.text&.strip
126
+ text ? text.to_f : 0.5
127
+ end
128
+
129
+ def parse_date(url_node)
130
+ raw_date = raw_date_from(url_node)
131
+ return unless raw_date
132
+
133
+ Time.parse(raw_date).utc.iso8601
134
+ rescue ArgumentError
135
+ nil
136
+ end
137
+
138
+ def raw_date_from(url_node)
139
+ news_node = url_node.at_xpath('.//news:publication_date', 'news' => NEWS_NS)
140
+ lastmod_node = url_node.at_xpath('./*[local-name()="lastmod"]')
141
+ (news_node || lastmod_node)&.text&.strip
142
+ end
143
+
144
+ def parse_title(url_node)
145
+ node = url_node.at_xpath('.//news:title', 'news' => NEWS_NS)
146
+ node&.text&.strip
147
+ end
148
+
149
+ def stale_date?(published_at)
150
+ return false unless published_at
151
+
152
+ time = Time.parse(published_at)
153
+ (Time.now.utc - time) > (max_age_days * 86_400)
154
+ rescue ArgumentError
155
+ false
156
+ end
157
+ end
158
+ end
159
+ end
160
+ end
161
+ end
@@ -42,14 +42,16 @@ module Html2rss
42
42
  # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
43
43
  # @param url [String, Html2rss::Url] canonical page URL
44
44
  # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
45
+ # @param body [String, nil] raw response body for direct sitemap XML parsing
45
46
  # @option _opts [Float] :min_priority minimum priority score (default: 0.3)
46
47
  # @option _opts [Integer] :max_age_days maximum entry age in days (default: 30)
47
- def initialize(parsed_body, url:, request_session: nil, **opts)
48
+ def initialize(parsed_body, url:, request_session: nil, body: nil, **opts)
48
49
  @parsed_body = parsed_body
50
+ @body = body
49
51
  @url = Html2rss::Url.from_absolute(url)
50
52
  @request_session = request_session
51
- @min_priority = opts.fetch(:min_priority, Discovery::Sitemap::DEFAULT_MIN_PRIORITY)
52
- @max_age_days = opts.fetch(:max_age_days, Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS)
53
+ @min_priority = opts.fetch(:min_priority, Scraper::Sitemap::Parser::DEFAULT_MIN_PRIORITY)
54
+ @max_age_days = opts.fetch(:max_age_days, Scraper::Sitemap::Parser::DEFAULT_MAX_AGE_DAYS)
53
55
  end
54
56
 
55
57
  ##
@@ -71,7 +73,7 @@ module Html2rss
71
73
 
72
74
  private
73
75
 
74
- attr_reader :parsed_body, :url, :request_session, :min_priority, :max_age_days
76
+ attr_reader :parsed_body, :body, :url, :request_session, :min_priority, :max_age_days
75
77
 
76
78
  def fetch_and_parse_entries
77
79
  entries = parse_direct_sitemap
@@ -87,16 +89,17 @@ module Html2rss
87
89
  end
88
90
 
89
91
  def parse_sitemap_xml(xml_body)
90
- result = Discovery::Sitemap.call(xml_body, min_priority:, max_age_days:)
92
+ result = Scraper::Sitemap::Parser.call(xml_body, min_priority:, max_age_days:)
91
93
  return fetch_sub_sitemaps(result.sub_sitemap_urls) if result.sub_sitemap_urls.any?
92
94
 
93
95
  result.entries
94
96
  end
95
97
 
96
98
  def parse_direct_sitemap
99
+ return [] unless body
97
100
  return [] unless parsed_body.at_xpath('//*[local-name()="urlset" or local-name()="sitemapindex"]')
98
101
 
99
- Discovery::Sitemap.call(parsed_body.to_s, min_priority:, max_age_days:).entries
102
+ Scraper::Sitemap::Parser.call(body, min_priority:, max_age_days:).entries
100
103
  end
101
104
 
102
105
  def fetch_sub_sitemaps(sub_urls)
@@ -114,7 +117,7 @@ module Html2rss
114
117
  xml = fetch_xml(sub_url)
115
118
  return [] unless xml
116
119
 
117
- Discovery::Sitemap.call(xml, min_priority:, max_age_days:).entries
120
+ Scraper::Sitemap::Parser.call(xml, min_priority:, max_age_days:).entries
118
121
  rescue Html2rss::RequestService::RequestBudgetExceeded
119
122
  Log.warn("#{self.class}: sitemap fan-out stopped — request budget exhausted")
120
123
  nil
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'cgi'
3
4
  require 'date'
4
- require 'nokogiri'
5
5
 
6
6
  module Html2rss
7
7
  class AutoSource
@@ -153,7 +153,11 @@ module Html2rss
153
153
  end
154
154
 
155
155
  def rendered_text(value)
156
- rendered_html(value)&.then { Nokogiri::HTML.fragment(_1).text.strip }
156
+ html = rendered_html(value)
157
+ return unless html
158
+
159
+ text = CGI.unescapeHTML(html.gsub(/<[^>]*>/m, ' ')).gsub(/\s+/, ' ').strip
160
+ text unless text.empty?
157
161
  end
158
162
 
159
163
  def rendered_html(value)
@@ -19,19 +19,23 @@ module Html2rss
19
19
  # Maximum visible text length tolerated for app-shell classification.
20
20
  APP_SHELL_MAX_VISIBLE_TEXT_LENGTH = 220
21
21
 
22
- # Ordered scraper classes considered during auto-source extraction.
23
- SCRAPERS = [
24
- WordpressApi,
25
- Sitemap,
26
- Schema,
27
- Microdata,
28
- Microformats2,
29
- JsonState,
30
- MetaOembed,
31
- SemanticHtml,
32
- Html
22
+ # Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
23
+ # Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
24
+ SCRAPER_TIERS = [
25
+ [Schema, Microdata, Microformats2, JsonState].freeze,
26
+ [WordpressApi, Sitemap, MetaOembed].freeze,
27
+ [SemanticHtml].freeze,
28
+ [Html].freeze
33
29
  ].freeze
34
30
 
31
+ # Flat ordered list (request slot accounting, detection helpers).
32
+ SCRAPERS = SCRAPER_TIERS.flatten.freeze
33
+
34
+ # Heuristic scrapers that share one memoized SST::Document per page.
35
+ HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
36
+ # Scrapers that accept a shared follow-up +request_session+.
37
+ REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
38
+
35
39
  ##
36
40
  # Error raised when no suitable scraper is found.
37
41
  class NoScraperFound < Html2rss::Error
@@ -80,6 +84,7 @@ module Html2rss
80
84
  # @option opts [Hash] :meta_oembed scraper toggle and configuration
81
85
  # @option opts [Hash] :semantic_html scraper toggle and configuration
82
86
  # @option opts [Hash] :html scraper toggle and configuration
87
+ # @option opts [Hash] :sitemap scraper toggle and configuration
83
88
  # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
84
89
  def self.from(parsed_body, opts = Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
85
90
  scrapers = SCRAPERS.select { |scraper| opts.dig(scraper.options_key, :enabled) }
@@ -90,11 +95,33 @@ module Html2rss
90
95
  scrapers
91
96
  end
92
97
 
93
- # Returns scraper instances ready for extraction.
94
- # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
95
- # @param url [String, Html2rss::Url] The page url.
96
- # @param request_session [Html2rss::RequestSession, nil] Shared follow-up session.
97
- # @param opts [Hash] The options hash.
98
+ ##
99
+ # @param tier [Array<Class>]
100
+ # @return [Boolean]
101
+ def self.heuristic_tier?(tier)
102
+ tier.intersect?(HEURISTIC_SCRAPERS)
103
+ end
104
+
105
+ ##
106
+ # @param parsed_body [Nokogiri::HTML::Document]
107
+ # @return [SST::Document, nil]
108
+ def self.normalize_sst(parsed_body)
109
+ SST::Normalizer.call(parsed_body)
110
+ rescue ArgumentError
111
+ nil
112
+ end
113
+
114
+ ##
115
+ # Builds a scraper when enabled; returns nil when disabled.
116
+ #
117
+ # @param scraper [Class]
118
+ # @param parsed_body [Nokogiri::HTML::Document]
119
+ # @param opts [Hash] full scraper options map
120
+ # @param url [Html2rss::Url, String]
121
+ # @param request_session [Html2rss::RequestSession, nil]
122
+ # @param body [String, nil]
123
+ # @param document [SST::Document, nil]
124
+ # @param link_resolver [Scoring::LinkResolver, nil]
98
125
  # @option opts [Hash] :wordpress_api scraper toggle and configuration
99
126
  # @option opts [Hash] :schema scraper toggle and configuration
100
127
  # @option opts [Hash] :microdata scraper toggle and configuration
@@ -103,50 +130,60 @@ module Html2rss
103
130
  # @option opts [Hash] :meta_oembed scraper toggle and configuration
104
131
  # @option opts [Hash] :semantic_html scraper toggle and configuration
105
132
  # @option opts [Hash] :html scraper toggle and configuration
106
- # @return [Array<Object>] An array of scraper instances that can handle the parsed body.
107
- #
108
- # `instances_for` is the main entrypoint for extraction. It lets a scraper
109
- # decide whether it matches using the same instance that will later yield
110
- # article hashes, which keeps precomputed state close to the scraper that
111
- # owns it.
112
- def self.instances_for(parsed_body, url:, request_session: nil,
113
- opts: Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
114
- instances = SCRAPERS.filter_map do |scraper|
115
- next unless opts.dig(scraper.options_key, :enabled)
116
-
117
- instance = scraper.new(parsed_body, url:, request_session:, **opts.fetch(scraper.options_key, {}))
118
- next unless extractable_instance?(instance, parsed_body)
119
-
120
- instance
121
- end
122
-
123
- raise no_scraper_found_for(parsed_body) if instances.empty?
124
-
125
- instances
133
+ # @option opts [Hash] :sitemap scraper toggle and configuration
134
+ # @return [Object, nil]
135
+ # rubocop:disable Metrics/ParameterLists -- construction context for structured and heuristic scrapers
136
+ def self.build_instance(scraper, parsed_body, opts:, url:, request_session: nil, body: nil, document: nil,
137
+ link_resolver: nil)
138
+ return unless opts.dig(scraper.options_key, :enabled)
139
+ return if HEURISTIC_SCRAPERS.include?(scraper) && document.nil?
140
+
141
+ scraper_opts = opts.fetch(scraper.options_key, {}).except(:enabled)
142
+ kwargs = construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
143
+ scraper.new(parsed_body, url:, **kwargs, **scraper_opts)
126
144
  end
145
+ # rubocop:enable Metrics/ParameterLists
127
146
 
147
+ ##
148
+ # @param instance [Object]
149
+ # @param parsed_body [Nokogiri::HTML::Document]
150
+ # @return [Boolean]
128
151
  def self.extractable_instance?(instance, parsed_body)
129
152
  return instance.extractable? if instance.respond_to?(:extractable?)
130
153
 
131
154
  instance.class.articles?(parsed_body)
132
155
  end
133
- private_class_method :extractable_instance?
134
156
 
135
- def self.no_scraper_found_for(parsed_body)
136
- NoScraperFound.new(category: classify_no_scraper_surface(parsed_body))
157
+ ##
158
+ # @param parsed_body [Nokogiri::HTML::Document]
159
+ # @param body [String, nil] raw response body (preferred for blocked-surface checks)
160
+ # @return [NoScraperFound]
161
+ def self.no_scraper_found_for(parsed_body, body: nil)
162
+ NoScraperFound.new(category: classify_no_scraper_surface(parsed_body, body:))
163
+ end
164
+
165
+ def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
166
+ if HEURISTIC_SCRAPERS.include?(scraper)
167
+ { document:, link_resolver: }.compact
168
+ else
169
+ {}.tap do |kwargs|
170
+ kwargs[:request_session] = request_session if REQUEST_SESSION_SCRAPERS.include?(scraper)
171
+ kwargs[:body] = body if scraper == Sitemap
172
+ end
173
+ end
137
174
  end
138
- private_class_method :no_scraper_found_for
175
+ private_class_method :construction_kwargs
139
176
 
140
- def self.classify_no_scraper_surface(parsed_body)
141
- return :blocked_surface if blocked_surface?(parsed_body)
177
+ def self.classify_no_scraper_surface(parsed_body, body: nil)
178
+ return :blocked_surface if blocked_surface?(parsed_body, body:)
142
179
  return :app_shell if app_shell_surface?(parsed_body)
143
180
 
144
181
  :unsupported_surface
145
182
  end
146
183
  private_class_method :classify_no_scraper_surface
147
184
 
148
- def self.blocked_surface?(parsed_body)
149
- Html2rss::RequestService::BlockedSurface.interstitial?(parsed_body.to_html)
185
+ def self.blocked_surface?(parsed_body, body: nil)
186
+ Html2rss::RequestService::BlockedSurface.interstitial?(body || parsed_body.to_html)
150
187
  end
151
188
  private_class_method :blocked_surface?
152
189
 
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ class AutoSource
5
+ ##
6
+ # One candidate content block discovered by {Segmenter}.
7
+ Segment = Data.define(:root_node, :primary_link, :strategy, :position) do
8
+ ##
9
+ # @param root_node [SST::Node]
10
+ # @param primary_link [SST::Node, nil]
11
+ # @param strategy [Symbol]
12
+ # @param position [Integer]
13
+ # @return [Segment]
14
+ # @raise [ArgumentError] on invalid construction
15
+ def self.build(root_node:, strategy:, position:, primary_link: nil)
16
+ raise ArgumentError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
17
+ unless primary_link.nil? || primary_link.is_a?(SST::Node)
18
+ raise ArgumentError, 'primary_link must be SST::Node or nil'
19
+ end
20
+ raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
21
+ raise ArgumentError, 'position must be Integer' unless position.is_a?(Integer)
22
+
23
+ new(root_node:, primary_link:, strategy:, position:)
24
+ end
25
+ end
26
+ # Allowed Segmenter strategies.
27
+ Segment::STRATEGIES = %i[semantic list cluster].to_set.freeze
28
+ end
29
+ end