html2rss 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/html2rss/auto_source/scraper/html.rb +68 -145
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +116 -0
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +63 -0
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +179 -0
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +55 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +0 -379
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +35 -30
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +40 -120
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +161 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +10 -7
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +6 -2
- data/lib/html2rss/auto_source/scraper.rb +81 -44
- data/lib/html2rss/auto_source/segment.rb +29 -0
- data/lib/html2rss/auto_source/segmenter/cluster.rb +163 -0
- data/lib/html2rss/auto_source/segmenter/list.rb +101 -0
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +81 -0
- data/lib/html2rss/auto_source/segmenter/semantic.rb +104 -0
- data/lib/html2rss/auto_source/segmenter.rb +66 -0
- data/lib/html2rss/auto_source.rb +74 -21
- data/lib/html2rss/cli.rb +8 -13
- data/lib/html2rss/config/auto_source_contract.rb +2 -0
- data/lib/html2rss/config/request_controls.rb +33 -0
- data/lib/html2rss/config/validator.rb +15 -8
- data/lib/html2rss/config.rb +6 -2
- data/lib/html2rss/feed_builder/rss.rb +17 -5
- data/lib/html2rss/feed_pipeline.rb +4 -0
- data/lib/html2rss/feed_result.rb +1 -1
- data/lib/html2rss/html/article_extractor/category_extractor.rb +10 -36
- data/lib/html2rss/html/article_extractor/date_extractor.rb +2 -7
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +5 -50
- data/lib/html2rss/html/article_extractor/image_extractor.rb +5 -16
- data/lib/html2rss/html/article_rules/category.rb +55 -0
- data/lib/html2rss/html/article_rules/date.rb +25 -0
- data/lib/html2rss/html/article_rules/enclosure.rb +72 -0
- data/lib/html2rss/html/article_rules/image.rb +109 -0
- data/lib/html2rss/html/article_rules.rb +10 -0
- data/lib/html2rss/html/rendering/audio_renderer.rb +2 -12
- data/lib/html2rss/html/rendering/escaped_attributes.rb +27 -0
- data/lib/html2rss/html/rendering/image_renderer.rb +2 -12
- data/lib/html2rss/html/rendering/pdf_renderer.rb +2 -8
- data/lib/html2rss/html/rendering/video_renderer.rb +2 -12
- data/lib/html2rss/html/sst_article_extractor.rb +279 -0
- data/lib/html2rss/link_destination/destination_facts.rb +40 -0
- data/lib/html2rss/link_destination/noise_policy.rb +79 -0
- data/lib/html2rss/link_destination/path_classifier.rb +205 -0
- data/lib/html2rss/link_destination/text_classifier.rb +64 -0
- data/lib/html2rss/link_destination.rb +8 -0
- data/lib/html2rss/request_service/faraday_strategy.rb +42 -1
- data/lib/html2rss/request_service/network_guard.rb +5 -3
- data/lib/html2rss/scoring/anchor_score.rb +34 -0
- data/lib/html2rss/scoring/cluster_scorer.rb +87 -0
- data/lib/html2rss/scoring/container_assessor.rb +83 -0
- data/lib/html2rss/scoring/engine.rb +101 -0
- data/lib/html2rss/scoring/link_resolver.rb +72 -0
- data/lib/html2rss/scoring/observation.rb +53 -0
- data/lib/html2rss/scoring/ranked_segment.rb +45 -0
- data/lib/html2rss/scoring/score.rb +30 -0
- data/lib/html2rss/scoring.rb +33 -0
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +10 -1
- data/lib/html2rss/selectors.rb +0 -20
- data/lib/html2rss/sst/attrs.rb +91 -0
- data/lib/html2rss/sst/document.rb +23 -0
- data/lib/html2rss/sst/index.rb +112 -0
- data/lib/html2rss/sst/node.rb +147 -0
- data/lib/html2rss/sst/normalizer.rb +171 -0
- data/lib/html2rss/sst/tags.rb +26 -0
- data/lib/html2rss/sst/text.rb +81 -0
- data/lib/html2rss/sst.rb +8 -0
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +26 -26
- data/schema/html2rss-config.schema.json +20 -0
- metadata +42 -18
- data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +0 -80
- data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +0 -86
- data/lib/html2rss/auto_source/discovery/dom_clustering.rb +0 -119
- data/lib/html2rss/auto_source/discovery/list_candidates.rb +0 -94
- data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +0 -219
- data/lib/html2rss/auto_source/discovery/semantic_containers.rb +0 -71
- data/lib/html2rss/auto_source/discovery/sitemap.rb +0 -159
- data/lib/html2rss/auto_source/discovery.rb +0 -14
- data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +0 -28
- data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +0 -106
- data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +0 -80
- data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +0 -42
- data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +0 -38
- data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +0 -221
- data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +0 -66
- data/lib/html2rss/auto_source/link_heuristics.rb +0 -139
|
@@ -4,31 +4,12 @@ module Html2rss
|
|
|
4
4
|
class AutoSource
|
|
5
5
|
module Scraper
|
|
6
6
|
##
|
|
7
|
-
# Scrapes semantic containers
|
|
8
|
-
# block before extraction.
|
|
9
|
-
#
|
|
10
|
-
# This scraper is intentionally container-first:
|
|
11
|
-
# 1. collect candidate semantic containers once
|
|
12
|
-
# 2. select the strongest content-like anchor within each container
|
|
13
|
-
# 3. extract fields from the container while honoring that anchor choice
|
|
14
|
-
#
|
|
15
|
-
# The result is lower recall on weak-signal blocks, but much better link
|
|
16
|
-
# quality on modern teaser cards that mix headlines, utility links, and
|
|
17
|
-
# duplicate image overlays.
|
|
7
|
+
# Scrapes semantic containers via SST: Normalizer → Segmenter → Scoring → SstArticleExtractor.
|
|
18
8
|
class SemanticHtml
|
|
19
9
|
include Enumerable
|
|
20
10
|
|
|
21
|
-
#
|
|
22
|
-
|
|
23
|
-
:container,
|
|
24
|
-
:selected_anchor,
|
|
25
|
-
:destination_facts,
|
|
26
|
-
:quality_score,
|
|
27
|
-
:junk_score,
|
|
28
|
-
:final_score,
|
|
29
|
-
:position,
|
|
30
|
-
:article
|
|
31
|
-
)
|
|
11
|
+
# Maximum articles materialized after ranking.
|
|
12
|
+
TOP_K = Scoring::Engine::TOP_K
|
|
32
13
|
|
|
33
14
|
##
|
|
34
15
|
# @return [Symbol] config key used to enable or configure this scraper
|
|
@@ -40,133 +21,72 @@ module Html2rss
|
|
|
40
21
|
return false unless parsed_body
|
|
41
22
|
|
|
42
23
|
new(parsed_body, url: 'https://example.com').extractable?
|
|
24
|
+
rescue ArgumentError
|
|
25
|
+
false
|
|
43
26
|
end
|
|
44
27
|
|
|
45
|
-
# @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
|
|
28
|
+
# @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
|
|
46
29
|
# @param url [String, Html2rss::Url] base url
|
|
47
|
-
# @param
|
|
30
|
+
# @param document [SST::Document, nil] memoized SST document from AutoSource
|
|
31
|
+
# @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
|
|
48
32
|
# @param opts [Hash] scraper-specific options
|
|
49
33
|
# @option opts [Boolean] :fallback_anchorless whether to keep containers without a primary anchor
|
|
50
|
-
def initialize(parsed_body, url:,
|
|
34
|
+
def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
|
|
51
35
|
@parsed_body = parsed_body
|
|
36
|
+
@provided_document = document
|
|
37
|
+
@provided_link_resolver = link_resolver
|
|
52
38
|
@url = url
|
|
53
|
-
@extractor = extractor
|
|
54
39
|
@permit_unanchored = opts.fetch(:fallback_anchorless, false)
|
|
55
|
-
@link_heuristics = LinkHeuristics.new(url)
|
|
56
40
|
end
|
|
57
41
|
|
|
58
|
-
attr_reader :parsed_body
|
|
59
|
-
|
|
60
42
|
##
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
# Detection and extraction share the same memoized entry list so this
|
|
65
|
-
# scraper does not rerun anchor ranking once a page has already been
|
|
66
|
-
# accepted as extractable.
|
|
67
|
-
#
|
|
68
|
-
# @yieldparam article_hash [Hash] extracted article hash
|
|
69
|
-
# @return [Enumerator<Hash>]
|
|
43
|
+
# @yieldparam article [Html2rss::Article]
|
|
44
|
+
# @return [Enumerator<Html2rss::Article>]
|
|
70
45
|
def each
|
|
71
46
|
return enum_for(:each) unless block_given?
|
|
72
47
|
|
|
73
|
-
|
|
48
|
+
articles.each { yield _1 }
|
|
74
49
|
end
|
|
75
50
|
|
|
76
51
|
##
|
|
77
|
-
#
|
|
78
|
-
# a selectable primary anchor.
|
|
79
|
-
#
|
|
80
|
-
# @return [Boolean] true when at least one candidate container yields a primary anchor
|
|
52
|
+
# @return [Boolean]
|
|
81
53
|
def extractable?
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# @return [Array<Nokogiri::XML::Node>]
|
|
86
|
-
def candidate_containers
|
|
87
|
-
@candidate_containers ||= collect_candidate_containers
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# @param container [Nokogiri::XML::Node]
|
|
91
|
-
# @return [Nokogiri::XML::Node, nil]
|
|
92
|
-
def primary_anchor_for(container)
|
|
93
|
-
Discovery::SemanticAnchorCandidates.new(
|
|
94
|
-
container,
|
|
95
|
-
link_heuristics: @link_heuristics
|
|
96
|
-
).to_a.max_by(&:score)&.anchor
|
|
54
|
+
ranked_segments.any?
|
|
55
|
+
rescue ArgumentError
|
|
56
|
+
false
|
|
97
57
|
end
|
|
98
58
|
|
|
99
|
-
|
|
100
|
-
def extractable_entries
|
|
101
|
-
@extractable_entries ||= candidate_containers.filter_map do |container|
|
|
102
|
-
selected_anchor = primary_anchor_for(container)
|
|
103
|
-
|
|
104
|
-
next unless selected_anchor || @permit_unanchored
|
|
105
|
-
|
|
106
|
-
destination_facts = selected_anchor ? normalized_destination(selected_anchor) : nil
|
|
107
|
-
next if selected_anchor && !destination_facts
|
|
108
|
-
# Cheap path-only reject before title/DOM hard-junk observations.
|
|
109
|
-
next if destination_facts&.high_confidence_junk_path
|
|
110
|
-
|
|
111
|
-
signals = @link_heuristics.assess_container(container, selected_anchor, destination_facts:)
|
|
112
|
-
next if signals.hard_junk?
|
|
59
|
+
private
|
|
113
60
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
junk_score: signals.junk_score,
|
|
120
|
-
final_score: signals.final_score,
|
|
121
|
-
position: document_position(container),
|
|
122
|
-
article: nil
|
|
123
|
-
)
|
|
61
|
+
def articles
|
|
62
|
+
@articles ||= begin
|
|
63
|
+
deduplicator = EntryDeduplicator.new(@url, scraper: self.class, link_resolver:)
|
|
64
|
+
entries = deduplicator.call(ranked_segments)
|
|
65
|
+
entries.filter_map { |ranked| deduplicator.article_for(ranked) }
|
|
124
66
|
end
|
|
67
|
+
rescue ArgumentError
|
|
68
|
+
[]
|
|
125
69
|
end
|
|
126
|
-
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
127
|
-
|
|
128
|
-
# rubocop:disable Metrics/MethodLength
|
|
129
|
-
def ranked_entries
|
|
130
|
-
@ranked_entries ||= begin
|
|
131
|
-
deduplicator = EntryDeduplicator.new(@url, @extractor)
|
|
132
|
-
entries = deduplicator.call(extractable_entries)
|
|
133
|
-
entries = stable_rank(entries)
|
|
134
|
-
|
|
135
|
-
entries.filter_map do |entry|
|
|
136
|
-
article = deduplicator.article_for(entry)
|
|
137
|
-
next unless article
|
|
138
70
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
end
|
|
71
|
+
def ranked_segments
|
|
72
|
+
@ranked_segments ||= begin
|
|
73
|
+
segments = Segmenter.call(
|
|
74
|
+
document,
|
|
75
|
+
base_url: @url,
|
|
76
|
+
strategy: :semantic,
|
|
77
|
+
permit_unanchored: @permit_unanchored,
|
|
78
|
+
link_resolver:
|
|
79
|
+
)
|
|
80
|
+
Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
|
|
150
81
|
end
|
|
151
82
|
end
|
|
152
|
-
# rubocop:enable Metrics/MethodLength
|
|
153
|
-
|
|
154
|
-
def collect_candidate_containers
|
|
155
|
-
Discovery::SemanticContainers.call(parsed_body)
|
|
156
|
-
end
|
|
157
|
-
|
|
158
|
-
private
|
|
159
|
-
|
|
160
|
-
def document_position(container)
|
|
161
|
-
(@document_positions ||= candidate_containers.each_with_index.to_h).fetch(container)
|
|
162
|
-
end
|
|
163
83
|
|
|
164
|
-
def
|
|
165
|
-
|
|
84
|
+
def link_resolver
|
|
85
|
+
@link_resolver ||= @provided_link_resolver || Scoring::LinkResolver.new(@url)
|
|
166
86
|
end
|
|
167
87
|
|
|
168
|
-
def
|
|
169
|
-
|
|
88
|
+
def document
|
|
89
|
+
@document ||= @provided_document || SST::Normalizer.call(@parsed_body)
|
|
170
90
|
end
|
|
171
91
|
end
|
|
172
92
|
end
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'date'
|
|
4
|
+
require 'nokogiri'
|
|
5
|
+
|
|
6
|
+
module Html2rss
|
|
7
|
+
class AutoSource
|
|
8
|
+
module Scraper
|
|
9
|
+
class Sitemap
|
|
10
|
+
##
|
|
11
|
+
# Parses XML sitemap documents and extracts prioritized, recent article entries.
|
|
12
|
+
class Parser
|
|
13
|
+
# Default minimum priority threshold (0.0 to 1.0)
|
|
14
|
+
DEFAULT_MIN_PRIORITY = 0.3
|
|
15
|
+
# Default maximum age in days for sitemap entries
|
|
16
|
+
DEFAULT_MAX_AGE_DAYS = 30
|
|
17
|
+
# Google News sitemap XML namespace
|
|
18
|
+
NEWS_NS = 'http://www.google.com/schemas/sitemap-news/0.9'
|
|
19
|
+
|
|
20
|
+
# Result value object returned by .call
|
|
21
|
+
# @!attribute entries [Array<SitemapEntry>] parsed entries (empty for sitemapindex docs)
|
|
22
|
+
# @!attribute sub_sitemap_urls [Array<String>] child sitemap URLs (empty for urlset docs)
|
|
23
|
+
Result = Data.define(:entries, :sub_sitemap_urls)
|
|
24
|
+
|
|
25
|
+
# Immutable facts for one sitemap entry
|
|
26
|
+
SitemapEntry = Data.define(:url, :title, :published_at, :priority, :changefreq)
|
|
27
|
+
|
|
28
|
+
class << self
|
|
29
|
+
##
|
|
30
|
+
# Parses sitemap XML body and returns a Result.
|
|
31
|
+
#
|
|
32
|
+
# @param xml_content [String] sitemap XML body string
|
|
33
|
+
# @param min_priority [Float] minimum priority score (0.0..1.0)
|
|
34
|
+
# @param max_age_days [Integer] maximum age in days for entries
|
|
35
|
+
# @return [Result] result with entries and/or sub_sitemap_urls
|
|
36
|
+
def call(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
|
|
37
|
+
new(xml_content, min_priority:, max_age_days:).call
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
##
|
|
42
|
+
# @param xml_content [String]
|
|
43
|
+
# @param min_priority [Float]
|
|
44
|
+
# @param max_age_days [Integer]
|
|
45
|
+
def initialize(xml_content, min_priority: DEFAULT_MIN_PRIORITY, max_age_days: DEFAULT_MAX_AGE_DAYS)
|
|
46
|
+
@xml_content = xml_content
|
|
47
|
+
@min_priority = min_priority
|
|
48
|
+
@max_age_days = max_age_days
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
##
|
|
52
|
+
# @return [Result]
|
|
53
|
+
def call
|
|
54
|
+
document = Nokogiri::XML(xml_content)
|
|
55
|
+
return Result.new(entries: [], sub_sitemap_urls: []) if invalid_xml?(document)
|
|
56
|
+
|
|
57
|
+
if sitemapindex?(document)
|
|
58
|
+
Result.new(entries: [], sub_sitemap_urls: extract_sub_sitemap_urls(document))
|
|
59
|
+
else
|
|
60
|
+
Result.new(entries: extract_entries(document), sub_sitemap_urls: [])
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
attr_reader :xml_content, :min_priority, :max_age_days
|
|
67
|
+
|
|
68
|
+
def invalid_xml?(document)
|
|
69
|
+
document.errors.any? && document.at_css('urlset, sitemapindex').nil?
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def sitemapindex?(document)
|
|
73
|
+
!document.at_xpath('//*[local-name()="sitemapindex"]').nil?
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def extract_sub_sitemap_urls(document)
|
|
77
|
+
document.xpath('//*[local-name()="sitemap"]/*[local-name()="loc"]').filter_map do |node|
|
|
78
|
+
text = node.text.strip
|
|
79
|
+
text unless text.empty?
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def extract_entries(document)
|
|
84
|
+
document.xpath('//*[local-name()="url"]').filter_map { build_entry(_1) }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def build_entry(url_node)
|
|
88
|
+
loc = parse_loc(url_node)
|
|
89
|
+
return unless loc
|
|
90
|
+
return if invalid_entry_signals?(url_node)
|
|
91
|
+
|
|
92
|
+
published_at = parse_date(url_node)
|
|
93
|
+
return if stale_date?(published_at)
|
|
94
|
+
|
|
95
|
+
build_entry_facts(url_node, loc, published_at)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def build_entry_facts(url_node, loc, published_at)
|
|
99
|
+
SitemapEntry.new(
|
|
100
|
+
url: loc,
|
|
101
|
+
title: parse_title(url_node),
|
|
102
|
+
published_at:,
|
|
103
|
+
priority: parse_priority(url_node),
|
|
104
|
+
changefreq: parse_changefreq(url_node)
|
|
105
|
+
)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def parse_loc(url_node)
|
|
109
|
+
node = url_node.at_xpath('./*[local-name()="loc"]')
|
|
110
|
+
text = node&.text&.strip
|
|
111
|
+
text unless text.to_s.empty?
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def parse_changefreq(url_node)
|
|
115
|
+
node = url_node.at_xpath('./*[local-name()="changefreq"]')
|
|
116
|
+
node ? node.text.strip.downcase : nil
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def invalid_entry_signals?(url_node)
|
|
120
|
+
parse_priority(url_node) < min_priority || parse_changefreq(url_node) == 'never'
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def parse_priority(url_node)
|
|
124
|
+
node = url_node.at_xpath('./*[local-name()="priority"]')
|
|
125
|
+
text = node&.text&.strip
|
|
126
|
+
text ? text.to_f : 0.5
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def parse_date(url_node)
|
|
130
|
+
raw_date = raw_date_from(url_node)
|
|
131
|
+
return unless raw_date
|
|
132
|
+
|
|
133
|
+
Time.parse(raw_date).utc.iso8601
|
|
134
|
+
rescue ArgumentError
|
|
135
|
+
nil
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def raw_date_from(url_node)
|
|
139
|
+
news_node = url_node.at_xpath('.//news:publication_date', 'news' => NEWS_NS)
|
|
140
|
+
lastmod_node = url_node.at_xpath('./*[local-name()="lastmod"]')
|
|
141
|
+
(news_node || lastmod_node)&.text&.strip
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def parse_title(url_node)
|
|
145
|
+
node = url_node.at_xpath('.//news:title', 'news' => NEWS_NS)
|
|
146
|
+
node&.text&.strip
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def stale_date?(published_at)
|
|
150
|
+
return false unless published_at
|
|
151
|
+
|
|
152
|
+
time = Time.parse(published_at)
|
|
153
|
+
(Time.now.utc - time) > (max_age_days * 86_400)
|
|
154
|
+
rescue ArgumentError
|
|
155
|
+
false
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
@@ -42,14 +42,16 @@ module Html2rss
|
|
|
42
42
|
# @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
|
|
43
43
|
# @param url [String, Html2rss::Url] canonical page URL
|
|
44
44
|
# @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
|
|
45
|
+
# @param body [String, nil] raw response body for direct sitemap XML parsing
|
|
45
46
|
# @option _opts [Float] :min_priority minimum priority score (default: 0.3)
|
|
46
47
|
# @option _opts [Integer] :max_age_days maximum entry age in days (default: 30)
|
|
47
|
-
def initialize(parsed_body, url:, request_session: nil, **opts)
|
|
48
|
+
def initialize(parsed_body, url:, request_session: nil, body: nil, **opts)
|
|
48
49
|
@parsed_body = parsed_body
|
|
50
|
+
@body = body
|
|
49
51
|
@url = Html2rss::Url.from_absolute(url)
|
|
50
52
|
@request_session = request_session
|
|
51
|
-
@min_priority = opts.fetch(:min_priority,
|
|
52
|
-
@max_age_days = opts.fetch(:max_age_days,
|
|
53
|
+
@min_priority = opts.fetch(:min_priority, Scraper::Sitemap::Parser::DEFAULT_MIN_PRIORITY)
|
|
54
|
+
@max_age_days = opts.fetch(:max_age_days, Scraper::Sitemap::Parser::DEFAULT_MAX_AGE_DAYS)
|
|
53
55
|
end
|
|
54
56
|
|
|
55
57
|
##
|
|
@@ -71,7 +73,7 @@ module Html2rss
|
|
|
71
73
|
|
|
72
74
|
private
|
|
73
75
|
|
|
74
|
-
attr_reader :parsed_body, :url, :request_session, :min_priority, :max_age_days
|
|
76
|
+
attr_reader :parsed_body, :body, :url, :request_session, :min_priority, :max_age_days
|
|
75
77
|
|
|
76
78
|
def fetch_and_parse_entries
|
|
77
79
|
entries = parse_direct_sitemap
|
|
@@ -87,16 +89,17 @@ module Html2rss
|
|
|
87
89
|
end
|
|
88
90
|
|
|
89
91
|
def parse_sitemap_xml(xml_body)
|
|
90
|
-
result =
|
|
92
|
+
result = Scraper::Sitemap::Parser.call(xml_body, min_priority:, max_age_days:)
|
|
91
93
|
return fetch_sub_sitemaps(result.sub_sitemap_urls) if result.sub_sitemap_urls.any?
|
|
92
94
|
|
|
93
95
|
result.entries
|
|
94
96
|
end
|
|
95
97
|
|
|
96
98
|
def parse_direct_sitemap
|
|
99
|
+
return [] unless body
|
|
97
100
|
return [] unless parsed_body.at_xpath('//*[local-name()="urlset" or local-name()="sitemapindex"]')
|
|
98
101
|
|
|
99
|
-
|
|
102
|
+
Scraper::Sitemap::Parser.call(body, min_priority:, max_age_days:).entries
|
|
100
103
|
end
|
|
101
104
|
|
|
102
105
|
def fetch_sub_sitemaps(sub_urls)
|
|
@@ -114,7 +117,7 @@ module Html2rss
|
|
|
114
117
|
xml = fetch_xml(sub_url)
|
|
115
118
|
return [] unless xml
|
|
116
119
|
|
|
117
|
-
|
|
120
|
+
Scraper::Sitemap::Parser.call(xml, min_priority:, max_age_days:).entries
|
|
118
121
|
rescue Html2rss::RequestService::RequestBudgetExceeded
|
|
119
122
|
Log.warn("#{self.class}: sitemap fan-out stopped — request budget exhausted")
|
|
120
123
|
nil
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'cgi'
|
|
3
4
|
require 'date'
|
|
4
|
-
require 'nokogiri'
|
|
5
5
|
|
|
6
6
|
module Html2rss
|
|
7
7
|
class AutoSource
|
|
@@ -153,7 +153,11 @@ module Html2rss
|
|
|
153
153
|
end
|
|
154
154
|
|
|
155
155
|
def rendered_text(value)
|
|
156
|
-
rendered_html(value)
|
|
156
|
+
html = rendered_html(value)
|
|
157
|
+
return unless html
|
|
158
|
+
|
|
159
|
+
text = CGI.unescapeHTML(html.gsub(/<[^>]*>/m, ' ')).gsub(/\s+/, ' ').strip
|
|
160
|
+
text unless text.empty?
|
|
157
161
|
end
|
|
158
162
|
|
|
159
163
|
def rendered_html(value)
|
|
@@ -19,19 +19,23 @@ module Html2rss
|
|
|
19
19
|
# Maximum visible text length tolerated for app-shell classification.
|
|
20
20
|
APP_SHELL_MAX_VISIBLE_TEXT_LENGTH = 220
|
|
21
21
|
|
|
22
|
-
#
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
JsonState,
|
|
30
|
-
MetaOembed,
|
|
31
|
-
SemanticHtml,
|
|
32
|
-
Html
|
|
22
|
+
# Extraction tiers: merge within a tier, then stop when AutoSource has enough articles.
|
|
23
|
+
# Heuristic scrapers are separate tiers so SemanticHtml can satisfy before Html runs.
|
|
24
|
+
SCRAPER_TIERS = [
|
|
25
|
+
[Schema, Microdata, Microformats2, JsonState].freeze,
|
|
26
|
+
[WordpressApi, Sitemap, MetaOembed].freeze,
|
|
27
|
+
[SemanticHtml].freeze,
|
|
28
|
+
[Html].freeze
|
|
33
29
|
].freeze
|
|
34
30
|
|
|
31
|
+
# Flat ordered list (request slot accounting, detection helpers).
|
|
32
|
+
SCRAPERS = SCRAPER_TIERS.flatten.freeze
|
|
33
|
+
|
|
34
|
+
# Heuristic scrapers that share one memoized SST::Document per page.
|
|
35
|
+
HEURISTIC_SCRAPERS = [SemanticHtml, Html].freeze
|
|
36
|
+
# Scrapers that accept a shared follow-up +request_session+.
|
|
37
|
+
REQUEST_SESSION_SCRAPERS = [WordpressApi, Sitemap, MetaOembed].freeze
|
|
38
|
+
|
|
35
39
|
##
|
|
36
40
|
# Error raised when no suitable scraper is found.
|
|
37
41
|
class NoScraperFound < Html2rss::Error
|
|
@@ -80,6 +84,7 @@ module Html2rss
|
|
|
80
84
|
# @option opts [Hash] :meta_oembed scraper toggle and configuration
|
|
81
85
|
# @option opts [Hash] :semantic_html scraper toggle and configuration
|
|
82
86
|
# @option opts [Hash] :html scraper toggle and configuration
|
|
87
|
+
# @option opts [Hash] :sitemap scraper toggle and configuration
|
|
83
88
|
# @return [Array<Class>] An array of scraper classes that can handle the parsed body.
|
|
84
89
|
def self.from(parsed_body, opts = Html2rss::AutoSource::DEFAULT_CONFIG[:scraper])
|
|
85
90
|
scrapers = SCRAPERS.select { |scraper| opts.dig(scraper.options_key, :enabled) }
|
|
@@ -90,11 +95,33 @@ module Html2rss
|
|
|
90
95
|
scrapers
|
|
91
96
|
end
|
|
92
97
|
|
|
93
|
-
|
|
94
|
-
# @param
|
|
95
|
-
# @
|
|
96
|
-
|
|
97
|
-
|
|
98
|
+
##
|
|
99
|
+
# @param tier [Array<Class>]
|
|
100
|
+
# @return [Boolean]
|
|
101
|
+
def self.heuristic_tier?(tier)
|
|
102
|
+
tier.intersect?(HEURISTIC_SCRAPERS)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
##
|
|
106
|
+
# @param parsed_body [Nokogiri::HTML::Document]
|
|
107
|
+
# @return [SST::Document, nil]
|
|
108
|
+
def self.normalize_sst(parsed_body)
|
|
109
|
+
SST::Normalizer.call(parsed_body)
|
|
110
|
+
rescue ArgumentError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
##
|
|
115
|
+
# Builds a scraper when enabled; returns nil when disabled.
|
|
116
|
+
#
|
|
117
|
+
# @param scraper [Class]
|
|
118
|
+
# @param parsed_body [Nokogiri::HTML::Document]
|
|
119
|
+
# @param opts [Hash] full scraper options map
|
|
120
|
+
# @param url [Html2rss::Url, String]
|
|
121
|
+
# @param request_session [Html2rss::RequestSession, nil]
|
|
122
|
+
# @param body [String, nil]
|
|
123
|
+
# @param document [SST::Document, nil]
|
|
124
|
+
# @param link_resolver [Scoring::LinkResolver, nil]
|
|
98
125
|
# @option opts [Hash] :wordpress_api scraper toggle and configuration
|
|
99
126
|
# @option opts [Hash] :schema scraper toggle and configuration
|
|
100
127
|
# @option opts [Hash] :microdata scraper toggle and configuration
|
|
@@ -103,50 +130,60 @@ module Html2rss
|
|
|
103
130
|
# @option opts [Hash] :meta_oembed scraper toggle and configuration
|
|
104
131
|
# @option opts [Hash] :semantic_html scraper toggle and configuration
|
|
105
132
|
# @option opts [Hash] :html scraper toggle and configuration
|
|
106
|
-
# @
|
|
107
|
-
#
|
|
108
|
-
#
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
instance = scraper.new(parsed_body, url:, request_session:, **opts.fetch(scraper.options_key, {}))
|
|
118
|
-
next unless extractable_instance?(instance, parsed_body)
|
|
119
|
-
|
|
120
|
-
instance
|
|
121
|
-
end
|
|
122
|
-
|
|
123
|
-
raise no_scraper_found_for(parsed_body) if instances.empty?
|
|
124
|
-
|
|
125
|
-
instances
|
|
133
|
+
# @option opts [Hash] :sitemap scraper toggle and configuration
|
|
134
|
+
# @return [Object, nil]
|
|
135
|
+
# rubocop:disable Metrics/ParameterLists -- construction context for structured and heuristic scrapers
|
|
136
|
+
def self.build_instance(scraper, parsed_body, opts:, url:, request_session: nil, body: nil, document: nil,
|
|
137
|
+
link_resolver: nil)
|
|
138
|
+
return unless opts.dig(scraper.options_key, :enabled)
|
|
139
|
+
return if HEURISTIC_SCRAPERS.include?(scraper) && document.nil?
|
|
140
|
+
|
|
141
|
+
scraper_opts = opts.fetch(scraper.options_key, {}).except(:enabled)
|
|
142
|
+
kwargs = construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
|
|
143
|
+
scraper.new(parsed_body, url:, **kwargs, **scraper_opts)
|
|
126
144
|
end
|
|
145
|
+
# rubocop:enable Metrics/ParameterLists
|
|
127
146
|
|
|
147
|
+
##
|
|
148
|
+
# @param instance [Object]
|
|
149
|
+
# @param parsed_body [Nokogiri::HTML::Document]
|
|
150
|
+
# @return [Boolean]
|
|
128
151
|
def self.extractable_instance?(instance, parsed_body)
|
|
129
152
|
return instance.extractable? if instance.respond_to?(:extractable?)
|
|
130
153
|
|
|
131
154
|
instance.class.articles?(parsed_body)
|
|
132
155
|
end
|
|
133
|
-
private_class_method :extractable_instance?
|
|
134
156
|
|
|
135
|
-
|
|
136
|
-
|
|
157
|
+
##
|
|
158
|
+
# @param parsed_body [Nokogiri::HTML::Document]
|
|
159
|
+
# @param body [String, nil] raw response body (preferred for blocked-surface checks)
|
|
160
|
+
# @return [NoScraperFound]
|
|
161
|
+
def self.no_scraper_found_for(parsed_body, body: nil)
|
|
162
|
+
NoScraperFound.new(category: classify_no_scraper_surface(parsed_body, body:))
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def self.construction_kwargs(scraper, request_session:, body:, document:, link_resolver:)
|
|
166
|
+
if HEURISTIC_SCRAPERS.include?(scraper)
|
|
167
|
+
{ document:, link_resolver: }.compact
|
|
168
|
+
else
|
|
169
|
+
{}.tap do |kwargs|
|
|
170
|
+
kwargs[:request_session] = request_session if REQUEST_SESSION_SCRAPERS.include?(scraper)
|
|
171
|
+
kwargs[:body] = body if scraper == Sitemap
|
|
172
|
+
end
|
|
173
|
+
end
|
|
137
174
|
end
|
|
138
|
-
private_class_method :
|
|
175
|
+
private_class_method :construction_kwargs
|
|
139
176
|
|
|
140
|
-
def self.classify_no_scraper_surface(parsed_body)
|
|
141
|
-
return :blocked_surface if blocked_surface?(parsed_body)
|
|
177
|
+
def self.classify_no_scraper_surface(parsed_body, body: nil)
|
|
178
|
+
return :blocked_surface if blocked_surface?(parsed_body, body:)
|
|
142
179
|
return :app_shell if app_shell_surface?(parsed_body)
|
|
143
180
|
|
|
144
181
|
:unsupported_surface
|
|
145
182
|
end
|
|
146
183
|
private_class_method :classify_no_scraper_surface
|
|
147
184
|
|
|
148
|
-
def self.blocked_surface?(parsed_body)
|
|
149
|
-
Html2rss::RequestService::BlockedSurface.interstitial?(parsed_body.to_html)
|
|
185
|
+
def self.blocked_surface?(parsed_body, body: nil)
|
|
186
|
+
Html2rss::RequestService::BlockedSurface.interstitial?(body || parsed_body.to_html)
|
|
150
187
|
end
|
|
151
188
|
private_class_method :blocked_surface?
|
|
152
189
|
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class AutoSource
|
|
5
|
+
##
|
|
6
|
+
# One candidate content block discovered by {Segmenter}.
|
|
7
|
+
Segment = Data.define(:root_node, :primary_link, :strategy, :position) do
|
|
8
|
+
##
|
|
9
|
+
# @param root_node [SST::Node]
|
|
10
|
+
# @param primary_link [SST::Node, nil]
|
|
11
|
+
# @param strategy [Symbol]
|
|
12
|
+
# @param position [Integer]
|
|
13
|
+
# @return [Segment]
|
|
14
|
+
# @raise [ArgumentError] on invalid construction
|
|
15
|
+
def self.build(root_node:, strategy:, position:, primary_link: nil)
|
|
16
|
+
raise ArgumentError, 'root_node must be SST::Node' unless root_node.is_a?(SST::Node)
|
|
17
|
+
unless primary_link.nil? || primary_link.is_a?(SST::Node)
|
|
18
|
+
raise ArgumentError, 'primary_link must be SST::Node or nil'
|
|
19
|
+
end
|
|
20
|
+
raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless Segment::STRATEGIES.include?(strategy)
|
|
21
|
+
raise ArgumentError, 'position must be Integer' unless position.is_a?(Integer)
|
|
22
|
+
|
|
23
|
+
new(root_node:, primary_link:, strategy:, position:)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
# Allowed Segmenter strategies.
|
|
27
|
+
Segment::STRATEGIES = %i[semantic list cluster].to_set.freeze
|
|
28
|
+
end
|
|
29
|
+
end
|