html2rss 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/html2rss/article.rb +42 -35
- data/lib/html2rss/auto_source/scraper/html.rb +68 -145
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +116 -0
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +63 -0
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +179 -0
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +55 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +0 -379
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +29 -114
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +50 -11
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +14 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +61 -18
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +35 -30
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +40 -120
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +161 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +10 -7
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +6 -2
- data/lib/html2rss/auto_source/scraper.rb +81 -44
- data/lib/html2rss/auto_source/segment.rb +29 -0
- data/lib/html2rss/auto_source/segmenter/cluster.rb +163 -0
- data/lib/html2rss/auto_source/segmenter/list.rb +101 -0
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +81 -0
- data/lib/html2rss/auto_source/segmenter/semantic.rb +104 -0
- data/lib/html2rss/auto_source/segmenter.rb +66 -0
- data/lib/html2rss/auto_source.rb +74 -21
- data/lib/html2rss/channel.rb +159 -79
- data/lib/html2rss/cli.rb +8 -13
- data/lib/html2rss/config/auto_source_contract.rb +2 -0
- data/lib/html2rss/config/request_controls.rb +33 -0
- data/lib/html2rss/config/validator.rb +15 -8
- data/lib/html2rss/config.rb +6 -2
- data/lib/html2rss/feed_builder/item_presentation.rb +42 -0
- data/lib/html2rss/feed_builder/json_feed/item.rb +20 -4
- data/lib/html2rss/feed_builder/json_feed.rb +22 -2
- data/lib/html2rss/feed_builder/rss.rb +36 -24
- data/lib/html2rss/feed_builder.rb +9 -17
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +71 -28
- data/lib/html2rss/feed_pipeline.rb +66 -55
- data/lib/html2rss/feed_result.rb +92 -0
- data/lib/html2rss/html/article_extractor/category_extractor.rb +10 -36
- data/lib/html2rss/html/article_extractor/date_extractor.rb +2 -7
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +5 -50
- data/lib/html2rss/html/article_extractor/image_extractor.rb +5 -16
- data/lib/html2rss/html/article_rules/category.rb +55 -0
- data/lib/html2rss/html/article_rules/date.rb +25 -0
- data/lib/html2rss/html/article_rules/enclosure.rb +72 -0
- data/lib/html2rss/html/article_rules/image.rb +109 -0
- data/lib/html2rss/html/article_rules.rb +10 -0
- data/lib/html2rss/html/rendering/audio_renderer.rb +2 -12
- data/lib/html2rss/html/rendering/escaped_attributes.rb +27 -0
- data/lib/html2rss/html/rendering/image_renderer.rb +2 -12
- data/lib/html2rss/html/rendering/pdf_renderer.rb +2 -8
- data/lib/html2rss/html/rendering/video_renderer.rb +2 -12
- data/lib/html2rss/html/sst_article_extractor.rb +279 -0
- data/lib/html2rss/link_destination/destination_facts.rb +40 -0
- data/lib/html2rss/link_destination/noise_policy.rb +79 -0
- data/lib/html2rss/link_destination/path_classifier.rb +205 -0
- data/lib/html2rss/link_destination/text_classifier.rb +64 -0
- data/lib/html2rss/link_destination.rb +8 -0
- data/lib/html2rss/request_service/botasaurus_contract.rb +33 -5
- data/lib/html2rss/request_service/botasaurus_strategy.rb +13 -4
- data/lib/html2rss/request_service/budget.rb +7 -2
- data/lib/html2rss/request_service/faraday_strategy.rb +42 -1
- data/lib/html2rss/request_service/network_guard.rb +5 -3
- data/lib/html2rss/request_service/response.rb +9 -1
- data/lib/html2rss/request_service/strategy.rb +22 -0
- data/lib/html2rss/request_session.rb +4 -0
- data/lib/html2rss/scoring/anchor_score.rb +34 -0
- data/lib/html2rss/scoring/cluster_scorer.rb +87 -0
- data/lib/html2rss/scoring/container_assessor.rb +83 -0
- data/lib/html2rss/scoring/engine.rb +101 -0
- data/lib/html2rss/scoring/link_resolver.rb +72 -0
- data/lib/html2rss/scoring/observation.rb +53 -0
- data/lib/html2rss/scoring/ranked_segment.rb +45 -0
- data/lib/html2rss/scoring/score.rb +30 -0
- data/lib/html2rss/scoring.rb +33 -0
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +10 -1
- data/lib/html2rss/selectors.rb +26 -24
- data/lib/html2rss/sst/attrs.rb +91 -0
- data/lib/html2rss/sst/document.rb +23 -0
- data/lib/html2rss/sst/index.rb +112 -0
- data/lib/html2rss/sst/node.rb +147 -0
- data/lib/html2rss/sst/normalizer.rb +171 -0
- data/lib/html2rss/sst/tags.rb +26 -0
- data/lib/html2rss/sst/text.rb +81 -0
- data/lib/html2rss/sst.rb +8 -0
- data/lib/html2rss/status.rb +132 -0
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +41 -29
- data/schema/html2rss-config.schema.json +20 -0
- metadata +45 -19
- data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +0 -80
- data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +0 -86
- data/lib/html2rss/auto_source/discovery/dom_clustering.rb +0 -119
- data/lib/html2rss/auto_source/discovery/list_candidates.rb +0 -94
- data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +0 -219
- data/lib/html2rss/auto_source/discovery/semantic_containers.rb +0 -71
- data/lib/html2rss/auto_source/discovery/sitemap.rb +0 -159
- data/lib/html2rss/auto_source/discovery.rb +0 -14
- data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +0 -28
- data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +0 -106
- data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +0 -80
- data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +0 -42
- data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +0 -38
- data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +0 -221
- data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +0 -66
- data/lib/html2rss/auto_source/link_heuristics.rb +0 -139
- data/lib/html2rss/auto_source/scraper/schema/list_item.rb +0 -28
data/lib/html2rss.rb
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
require 'zeitwerk'
|
|
4
4
|
|
|
5
5
|
loader = Zeitwerk::Loader.for_gem
|
|
6
|
-
loader.inflector.inflect(
|
|
6
|
+
loader.inflector.inflect(
|
|
7
|
+
'cli' => 'CLI',
|
|
8
|
+
'sst' => 'SST'
|
|
9
|
+
)
|
|
7
10
|
loader.setup
|
|
8
11
|
|
|
9
12
|
require 'logger'
|
|
@@ -34,28 +37,43 @@ module Html2rss
|
|
|
34
37
|
Config.load_yaml(file, feed_name)
|
|
35
38
|
end
|
|
36
39
|
|
|
40
|
+
##
|
|
41
|
+
# Returns an opaque, Marshal-cacheable result of one scrape.
|
|
42
|
+
#
|
|
43
|
+
# Prefer this when the same scrape must render as RSS and JSON Feed (e.g. web cache).
|
|
44
|
+
#
|
|
45
|
+
# @param raw_config [Hash{Symbol => Object}] feed configuration
|
|
46
|
+
# @return [Html2rss::FeedResult]
|
|
47
|
+
def self.feed_result(raw_config)
|
|
48
|
+
FeedPipeline.new(raw_config).to_result
|
|
49
|
+
end
|
|
50
|
+
|
|
37
51
|
##
|
|
38
52
|
# Returns an RSS object generated from the provided configuration.
|
|
39
53
|
#
|
|
40
54
|
# @param raw_config [Hash{Symbol => Object}] feed configuration
|
|
41
55
|
# @return [RSS::Rss] generated RSS feed
|
|
42
56
|
def self.feed(raw_config)
|
|
43
|
-
|
|
57
|
+
feed_result(raw_config).to_rss
|
|
44
58
|
end
|
|
45
59
|
|
|
46
60
|
##
|
|
47
61
|
# Returns a JSONFeed 1.1 hash generated from the provided configuration.
|
|
48
62
|
#
|
|
49
63
|
# @param raw_config [Hash{Symbol => Object}] feed configuration
|
|
64
|
+
# @param feed_url [String, nil] optional self URL for the feed (JSON Feed +feed_url+)
|
|
50
65
|
# @return [Hash] JSONFeed-compliant hash
|
|
51
|
-
def self.json_feed(raw_config)
|
|
52
|
-
|
|
66
|
+
def self.json_feed(raw_config, feed_url: nil)
|
|
67
|
+
feed_result(raw_config).to_json_feed(feed_url:)
|
|
53
68
|
end
|
|
54
69
|
|
|
55
70
|
# rubocop:disable Metrics/ParameterLists
|
|
56
71
|
|
|
57
72
|
##
|
|
58
|
-
# Scrapes the provided URL and returns an RSS object.
|
|
73
|
+
# Scrapes the provided URL without hand-written selectors and returns an RSS object.
|
|
74
|
+
#
|
|
75
|
+
# Builds an auto_source config, then FeedPipeline runs structured scrapers and
|
|
76
|
+
# (when needed) the SST heuristic path — see docs/auto_source.md.
|
|
59
77
|
#
|
|
60
78
|
# @param url [String] source page URL
|
|
61
79
|
# @param strategy [Symbol] request strategy to use
|
|
@@ -63,18 +81,23 @@ module Html2rss
|
|
|
63
81
|
# @param max_redirects [Integer, nil] optional redirect limit override
|
|
64
82
|
# @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
|
|
65
83
|
# @param local_file_path [String, nil] optional local HTML file path
|
|
84
|
+
# @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
|
|
66
85
|
# @return [RSS::Rss] generated RSS feed
|
|
67
86
|
def self.auto_source(url,
|
|
68
87
|
strategy: :auto,
|
|
69
88
|
items_selector: nil,
|
|
70
89
|
max_redirects: nil,
|
|
71
90
|
max_requests: 4,
|
|
72
|
-
local_file_path: nil
|
|
73
|
-
|
|
91
|
+
local_file_path: nil,
|
|
92
|
+
limit: nil)
|
|
93
|
+
feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
|
|
94
|
+
local_file_path:, limit:))
|
|
74
95
|
end
|
|
75
96
|
|
|
76
97
|
##
|
|
77
|
-
# Scrapes the provided URL and returns a JSONFeed 1.1 hash.
|
|
98
|
+
# Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
|
|
99
|
+
#
|
|
100
|
+
# Same auto_source pipeline as {.auto_source}; see docs/auto_source.md.
|
|
78
101
|
#
|
|
79
102
|
# @param url [String] source page URL
|
|
80
103
|
# @param strategy [Symbol] request strategy to use
|
|
@@ -82,15 +105,17 @@ module Html2rss
|
|
|
82
105
|
# @param max_redirects [Integer, nil] optional redirect limit override
|
|
83
106
|
# @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
|
|
84
107
|
# @param local_file_path [String, nil] optional local HTML file path
|
|
108
|
+
# @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
|
|
85
109
|
# @return [Hash] JSONFeed-compliant hash
|
|
86
110
|
def self.auto_json_feed(url,
|
|
87
111
|
strategy: :auto,
|
|
88
112
|
items_selector: nil,
|
|
89
113
|
max_redirects: nil,
|
|
90
114
|
max_requests: 4,
|
|
91
|
-
local_file_path: nil
|
|
115
|
+
local_file_path: nil,
|
|
116
|
+
limit: nil)
|
|
92
117
|
json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
|
|
93
|
-
local_file_path:))
|
|
118
|
+
local_file_path:, limit:))
|
|
94
119
|
end
|
|
95
120
|
|
|
96
121
|
# rubocop:enable Metrics/ParameterLists
|
|
@@ -142,11 +167,14 @@ module Html2rss
|
|
|
142
167
|
class << self
|
|
143
168
|
private
|
|
144
169
|
|
|
145
|
-
|
|
170
|
+
# rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
|
|
171
|
+
def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
|
|
172
|
+
local_file_path: nil, limit: nil)
|
|
146
173
|
config = Config.auto_source_config(
|
|
147
174
|
url:,
|
|
148
175
|
items_selector:,
|
|
149
|
-
request_controls:
|
|
176
|
+
request_controls: Config::RequestControls.from_shortcut(strategy:, max_redirects:, max_requests:),
|
|
177
|
+
limit:
|
|
150
178
|
)
|
|
151
179
|
if local_file_path
|
|
152
180
|
config[:request] ||= {}
|
|
@@ -154,23 +182,7 @@ module Html2rss
|
|
|
154
182
|
end
|
|
155
183
|
config
|
|
156
184
|
end
|
|
157
|
-
|
|
158
|
-
def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
|
|
159
|
-
Config::RequestControls.new(
|
|
160
|
-
strategy:,
|
|
161
|
-
max_redirects:,
|
|
162
|
-
max_requests:,
|
|
163
|
-
explicit_keys: explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
|
|
164
|
-
)
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
|
|
168
|
-
keys = []
|
|
169
|
-
keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
|
|
170
|
-
keys << :max_redirects unless max_redirects.nil?
|
|
171
|
-
keys << :max_requests unless max_requests.nil?
|
|
172
|
-
keys
|
|
173
|
-
end
|
|
185
|
+
# rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
|
|
174
186
|
end
|
|
175
187
|
|
|
176
188
|
logger.level = defaults.log_level if logger.respond_to?(:level=)
|
|
@@ -45,6 +45,18 @@
|
|
|
45
45
|
"null",
|
|
46
46
|
"string"
|
|
47
47
|
]
|
|
48
|
+
},
|
|
49
|
+
"author": {
|
|
50
|
+
"type": [
|
|
51
|
+
"null",
|
|
52
|
+
"string"
|
|
53
|
+
]
|
|
54
|
+
},
|
|
55
|
+
"image": {
|
|
56
|
+
"type": [
|
|
57
|
+
"null",
|
|
58
|
+
"string"
|
|
59
|
+
]
|
|
48
60
|
}
|
|
49
61
|
},
|
|
50
62
|
"required": [
|
|
@@ -94,6 +106,13 @@
|
|
|
94
106
|
"$schema": "http://json-schema.org/draft-06/schema#",
|
|
95
107
|
"type": "object",
|
|
96
108
|
"properties": {
|
|
109
|
+
"limit": {
|
|
110
|
+
"type": "integer",
|
|
111
|
+
"not": {
|
|
112
|
+
"type": "null"
|
|
113
|
+
},
|
|
114
|
+
"exclusiveMinimum": 0
|
|
115
|
+
},
|
|
97
116
|
"scraper": {
|
|
98
117
|
"type": "object",
|
|
99
118
|
"properties": {
|
|
@@ -269,6 +288,7 @@
|
|
|
269
288
|
},
|
|
270
289
|
"required": [],
|
|
271
290
|
"default": {
|
|
291
|
+
"limit": 25,
|
|
272
292
|
"scraper": {
|
|
273
293
|
"wordpress_api": {
|
|
274
294
|
"enabled": true
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html2rss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.25.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gil Desmarais
|
|
@@ -278,40 +278,34 @@ files:
|
|
|
278
278
|
- lib/html2rss/article/enclosure.rb
|
|
279
279
|
- lib/html2rss/auto_source.rb
|
|
280
280
|
- lib/html2rss/auto_source/cleanup.rb
|
|
281
|
-
- lib/html2rss/auto_source/discovery.rb
|
|
282
|
-
- lib/html2rss/auto_source/discovery/dom_clustering.rb
|
|
283
|
-
- lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb
|
|
284
|
-
- lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb
|
|
285
|
-
- lib/html2rss/auto_source/discovery/list_candidates.rb
|
|
286
|
-
- lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb
|
|
287
|
-
- lib/html2rss/auto_source/discovery/semantic_containers.rb
|
|
288
|
-
- lib/html2rss/auto_source/discovery/sitemap.rb
|
|
289
|
-
- lib/html2rss/auto_source/link_heuristics.rb
|
|
290
|
-
- lib/html2rss/auto_source/link_heuristics/anchor_signals.rb
|
|
291
|
-
- lib/html2rss/auto_source/link_heuristics/container_assessor.rb
|
|
292
|
-
- lib/html2rss/auto_source/link_heuristics/container_signals.rb
|
|
293
|
-
- lib/html2rss/auto_source/link_heuristics/destination_facts.rb
|
|
294
|
-
- lib/html2rss/auto_source/link_heuristics/href_extractor.rb
|
|
295
|
-
- lib/html2rss/auto_source/link_heuristics/path_classifier.rb
|
|
296
|
-
- lib/html2rss/auto_source/link_heuristics/text_classifier.rb
|
|
297
281
|
- lib/html2rss/auto_source/scraper.rb
|
|
298
282
|
- lib/html2rss/auto_source/scraper/html.rb
|
|
299
283
|
- lib/html2rss/auto_source/scraper/json_state.rb
|
|
284
|
+
- lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb
|
|
285
|
+
- lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb
|
|
286
|
+
- lib/html2rss/auto_source/scraper/json_state/document_scanner.rb
|
|
287
|
+
- lib/html2rss/auto_source/scraper/json_state/value_finder.rb
|
|
300
288
|
- lib/html2rss/auto_source/scraper/meta_oembed.rb
|
|
301
289
|
- lib/html2rss/auto_source/scraper/microdata.rb
|
|
302
290
|
- lib/html2rss/auto_source/scraper/microformats2.rb
|
|
303
291
|
- lib/html2rss/auto_source/scraper/schema.rb
|
|
304
292
|
- lib/html2rss/auto_source/scraper/schema/category_extractor.rb
|
|
305
293
|
- lib/html2rss/auto_source/scraper/schema/item_list.rb
|
|
306
|
-
- lib/html2rss/auto_source/scraper/schema/list_item.rb
|
|
307
294
|
- lib/html2rss/auto_source/scraper/schema/thing.rb
|
|
308
295
|
- lib/html2rss/auto_source/scraper/semantic_html.rb
|
|
309
296
|
- lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb
|
|
310
297
|
- lib/html2rss/auto_source/scraper/sitemap.rb
|
|
298
|
+
- lib/html2rss/auto_source/scraper/sitemap/parser.rb
|
|
311
299
|
- lib/html2rss/auto_source/scraper/wordpress_api.rb
|
|
312
300
|
- lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
|
|
313
301
|
- lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb
|
|
314
302
|
- lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
|
|
303
|
+
- lib/html2rss/auto_source/segment.rb
|
|
304
|
+
- lib/html2rss/auto_source/segmenter.rb
|
|
305
|
+
- lib/html2rss/auto_source/segmenter/cluster.rb
|
|
306
|
+
- lib/html2rss/auto_source/segmenter/list.rb
|
|
307
|
+
- lib/html2rss/auto_source/segmenter/primary_link.rb
|
|
308
|
+
- lib/html2rss/auto_source/segmenter/semantic.rb
|
|
315
309
|
- lib/html2rss/channel.rb
|
|
316
310
|
- lib/html2rss/cli.rb
|
|
317
311
|
- lib/html2rss/config.rb
|
|
@@ -326,6 +320,7 @@ files:
|
|
|
326
320
|
- lib/html2rss/defaults.rb
|
|
327
321
|
- lib/html2rss/error.rb
|
|
328
322
|
- lib/html2rss/feed_builder.rb
|
|
323
|
+
- lib/html2rss/feed_builder/item_presentation.rb
|
|
329
324
|
- lib/html2rss/feed_builder/json_feed.rb
|
|
330
325
|
- lib/html2rss/feed_builder/json_feed/item.rb
|
|
331
326
|
- lib/html2rss/feed_builder/rss.rb
|
|
@@ -334,6 +329,7 @@ files:
|
|
|
334
329
|
- lib/html2rss/feed_pipeline/auto_fallback.rb
|
|
335
330
|
- lib/html2rss/feed_pipeline/runtime_policy.rb
|
|
336
331
|
- lib/html2rss/feed_pipeline/strategy_plan.rb
|
|
332
|
+
- lib/html2rss/feed_result.rb
|
|
337
333
|
- lib/html2rss/hash_util.rb
|
|
338
334
|
- lib/html2rss/html.rb
|
|
339
335
|
- lib/html2rss/html/article_extractor.rb
|
|
@@ -343,15 +339,27 @@ files:
|
|
|
343
339
|
- lib/html2rss/html/article_extractor/heading_extractor.rb
|
|
344
340
|
- lib/html2rss/html/article_extractor/id_generator.rb
|
|
345
341
|
- lib/html2rss/html/article_extractor/image_extractor.rb
|
|
342
|
+
- lib/html2rss/html/article_rules.rb
|
|
343
|
+
- lib/html2rss/html/article_rules/category.rb
|
|
344
|
+
- lib/html2rss/html/article_rules/date.rb
|
|
345
|
+
- lib/html2rss/html/article_rules/enclosure.rb
|
|
346
|
+
- lib/html2rss/html/article_rules/image.rb
|
|
346
347
|
- lib/html2rss/html/navigator.rb
|
|
347
348
|
- lib/html2rss/html/navigator/text_extractor.rb
|
|
348
349
|
- lib/html2rss/html/rendering.rb
|
|
349
350
|
- lib/html2rss/html/rendering/audio_renderer.rb
|
|
350
351
|
- lib/html2rss/html/rendering/description_builder.rb
|
|
352
|
+
- lib/html2rss/html/rendering/escaped_attributes.rb
|
|
351
353
|
- lib/html2rss/html/rendering/image_renderer.rb
|
|
352
354
|
- lib/html2rss/html/rendering/media_renderer.rb
|
|
353
355
|
- lib/html2rss/html/rendering/pdf_renderer.rb
|
|
354
356
|
- lib/html2rss/html/rendering/video_renderer.rb
|
|
357
|
+
- lib/html2rss/html/sst_article_extractor.rb
|
|
358
|
+
- lib/html2rss/link_destination.rb
|
|
359
|
+
- lib/html2rss/link_destination/destination_facts.rb
|
|
360
|
+
- lib/html2rss/link_destination/noise_policy.rb
|
|
361
|
+
- lib/html2rss/link_destination/path_classifier.rb
|
|
362
|
+
- lib/html2rss/link_destination/text_classifier.rb
|
|
355
363
|
- lib/html2rss/request_service.rb
|
|
356
364
|
- lib/html2rss/request_service/blocked_surface.rb
|
|
357
365
|
- lib/html2rss/request_service/botasaurus_contract.rb
|
|
@@ -377,6 +385,15 @@ files:
|
|
|
377
385
|
- lib/html2rss/request_session/pager/offset.rb
|
|
378
386
|
- lib/html2rss/request_session/pager/rel_next.rb
|
|
379
387
|
- lib/html2rss/request_session/pager/url_template.rb
|
|
388
|
+
- lib/html2rss/scoring.rb
|
|
389
|
+
- lib/html2rss/scoring/anchor_score.rb
|
|
390
|
+
- lib/html2rss/scoring/cluster_scorer.rb
|
|
391
|
+
- lib/html2rss/scoring/container_assessor.rb
|
|
392
|
+
- lib/html2rss/scoring/engine.rb
|
|
393
|
+
- lib/html2rss/scoring/link_resolver.rb
|
|
394
|
+
- lib/html2rss/scoring/observation.rb
|
|
395
|
+
- lib/html2rss/scoring/ranked_segment.rb
|
|
396
|
+
- lib/html2rss/scoring/score.rb
|
|
380
397
|
- lib/html2rss/selectors.rb
|
|
381
398
|
- lib/html2rss/selectors/extractors.rb
|
|
382
399
|
- lib/html2rss/selectors/extractors/attribute.rb
|
|
@@ -398,6 +415,15 @@ files:
|
|
|
398
415
|
- lib/html2rss/selectors/post_processors/sanitize_html.rb
|
|
399
416
|
- lib/html2rss/selectors/post_processors/substring.rb
|
|
400
417
|
- lib/html2rss/selectors/post_processors/template.rb
|
|
418
|
+
- lib/html2rss/sst.rb
|
|
419
|
+
- lib/html2rss/sst/attrs.rb
|
|
420
|
+
- lib/html2rss/sst/document.rb
|
|
421
|
+
- lib/html2rss/sst/index.rb
|
|
422
|
+
- lib/html2rss/sst/node.rb
|
|
423
|
+
- lib/html2rss/sst/normalizer.rb
|
|
424
|
+
- lib/html2rss/sst/tags.rb
|
|
425
|
+
- lib/html2rss/sst/text.rb
|
|
426
|
+
- lib/html2rss/status.rb
|
|
401
427
|
- lib/html2rss/url.rb
|
|
402
428
|
- lib/html2rss/version.rb
|
|
403
429
|
- lib/tasks/config_schema.rake
|
|
@@ -407,7 +433,7 @@ licenses:
|
|
|
407
433
|
- MIT
|
|
408
434
|
metadata:
|
|
409
435
|
allowed_push_host: https://rubygems.org
|
|
410
|
-
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.
|
|
436
|
+
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.25.0
|
|
411
437
|
rubygems_mfa_required: 'true'
|
|
412
438
|
rdoc_options: []
|
|
413
439
|
require_paths:
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Html2rss
|
|
4
|
-
class AutoSource
|
|
5
|
-
module Discovery
|
|
6
|
-
class DomClustering
|
|
7
|
-
##
|
|
8
|
-
# Scores candidate DOM groups using heading, time, date, and word-count signals.
|
|
9
|
-
class GroupScorer
|
|
10
|
-
def initialize
|
|
11
|
-
@text_words = {}.compare_by_identity
|
|
12
|
-
@has_date = {}.compare_by_identity
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
# @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate DOM groups
|
|
16
|
-
# @return [Array<Nokogiri::XML::Node>] nodes from the highest-scoring group
|
|
17
|
-
def select_best_group(groups)
|
|
18
|
-
best_nodes = []
|
|
19
|
-
best_score = -1
|
|
20
|
-
|
|
21
|
-
groups.each_value do |nodes|
|
|
22
|
-
score = score_group(nodes)
|
|
23
|
-
next if score.negative?
|
|
24
|
-
|
|
25
|
-
(best_nodes = nodes) && (best_score = score) if score > best_score
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
best_nodes
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
# @param nodes [Array<Nokogiri::XML::Node>]
|
|
32
|
-
# @return [Float] average visible word count across nodes
|
|
33
|
-
def avg_words(nodes)
|
|
34
|
-
nodes.sum { |n| text_words(n) } / nodes.size.to_f
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
private
|
|
38
|
-
|
|
39
|
-
def score_group(nodes)
|
|
40
|
-
avg_w = avg_words(nodes)
|
|
41
|
-
return -1 if avg_w < 5
|
|
42
|
-
|
|
43
|
-
score = nodes.size + (avg_w / 5.0)
|
|
44
|
-
score += 20 if nodes_heading?(nodes)
|
|
45
|
-
score += 20 if nodes_time?(nodes)
|
|
46
|
-
score += 40 if nodes_date?(nodes)
|
|
47
|
-
score
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def nodes_heading?(nodes)
|
|
51
|
-
nodes.any? do |n|
|
|
52
|
-
n.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(',')) ||
|
|
53
|
-
n.at_css('.font-bold, .font-semibold')
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def nodes_time?(nodes)
|
|
58
|
-
nodes.any? { |n| n.at_css('time, [datetime]') }
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
def nodes_date?(nodes)
|
|
62
|
-
nodes.any? { |n| date?(n) }
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
def text_words(node)
|
|
66
|
-
@text_words[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
def date?(node)
|
|
70
|
-
@has_date[node] ||= begin
|
|
71
|
-
text = Html2rss::Html::Navigator.extract_visible_text(node).to_s
|
|
72
|
-
text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
|
|
73
|
-
text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
end
|
|
80
|
-
end
|
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Html2rss
|
|
4
|
-
class AutoSource
|
|
5
|
-
module Discovery
|
|
6
|
-
class DomClustering
|
|
7
|
-
##
|
|
8
|
-
# Filters layout wrapper groups and resolves 1-to-1 nested card wrappers.
|
|
9
|
-
class OverlapResolver
|
|
10
|
-
# @param layout_tags [Set<String>] tags treated as layout containers
|
|
11
|
-
# @param word_counter [#avg_words] averages visible word counts for nodes
|
|
12
|
-
def initialize(layout_tags:, word_counter:)
|
|
13
|
-
@layout_tags = layout_tags
|
|
14
|
-
@word_counter = word_counter
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
# Discard group A if any node of A contains > 1 node of another group B
|
|
18
|
-
#
|
|
19
|
-
# @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate groups
|
|
20
|
-
# @return [Hash{String => Array<Nokogiri::XML::Node>}] groups that are not layout wrappers
|
|
21
|
-
def filter_containers(groups)
|
|
22
|
-
groups.reject do |cls_a, nodes_a|
|
|
23
|
-
groups.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
# If group A contains group B with the same size, keep the real content card.
|
|
28
|
-
#
|
|
29
|
-
# @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate groups
|
|
30
|
-
# @return [Hash{String => Array<Nokogiri::XML::Node>}] groups after 1-to-1 resolution
|
|
31
|
-
def filter_1_to_1_overlap(groups)
|
|
32
|
-
discarded = Set.new
|
|
33
|
-
groups.each_key do |cls_a|
|
|
34
|
-
groups.each_key do |cls_b|
|
|
35
|
-
next if cls_a == cls_b || discarded.include?(cls_a) || discarded.include?(cls_b)
|
|
36
|
-
|
|
37
|
-
resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
groups.except(*discarded)
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
private
|
|
45
|
-
|
|
46
|
-
attr_reader :layout_tags, :word_counter
|
|
47
|
-
|
|
48
|
-
# rubocop:disable Metrics/MethodLength
|
|
49
|
-
def container_of?(nodes_a, nodes_b)
|
|
50
|
-
return false unless layout_tags.include?(nodes_b.first.name)
|
|
51
|
-
|
|
52
|
-
nodes_a.any? do |node_a|
|
|
53
|
-
count = 0
|
|
54
|
-
nodes_b.each do |node_b|
|
|
55
|
-
next if node_a == node_b
|
|
56
|
-
|
|
57
|
-
if Html2rss::Html::Navigator.descendant_of?(node_b, node_a)
|
|
58
|
-
count += 1
|
|
59
|
-
break if count > 1
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
count > 1
|
|
63
|
-
end
|
|
64
|
-
end
|
|
65
|
-
# rubocop:enable Metrics/MethodLength
|
|
66
|
-
|
|
67
|
-
def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
|
|
68
|
-
nodes_a = groups[cls_a]
|
|
69
|
-
nodes_b = groups[cls_b]
|
|
70
|
-
return if nodes_a.size != nodes_b.size
|
|
71
|
-
|
|
72
|
-
nested = nodes_a.zip(nodes_b).all? { |a, b| a != b && Html2rss::Html::Navigator.descendant_of?(b, a) }
|
|
73
|
-
return unless nested
|
|
74
|
-
|
|
75
|
-
discarded << (keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b)
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
def keep_descendant?(nodes_a, nodes_b)
|
|
79
|
-
word_counter.avg_words(nodes_b) >= 0.8 * word_counter.avg_words(nodes_a) &&
|
|
80
|
-
layout_tags.include?(nodes_b.first.name)
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
end
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Html2rss
|
|
4
|
-
class AutoSource
|
|
5
|
-
module Discovery
|
|
6
|
-
##
|
|
7
|
-
# Discovers repeated content card nodes in anchorless or classless DOM trees
|
|
8
|
-
# by evaluating class groups first and falling back to 1-level tag structure signatures.
|
|
9
|
-
class DomClustering
|
|
10
|
-
# Node tags considered layout containers
|
|
11
|
-
LAYOUT_TAG_NAMES = Set['div', 'section', 'article', 'li', 'ul', 'ol'].freeze
|
|
12
|
-
# HTML/layout tags excluded from candidate nodes (owned by Html::Navigator).
|
|
13
|
-
EXCLUDED_TAGS = Html2rss::Html::Navigator::CLUSTER_EXCLUDED_TAGS
|
|
14
|
-
|
|
15
|
-
class << self
|
|
16
|
-
##
|
|
17
|
-
# Clusters elements in parsed_body and returns the best set of content card nodes.
|
|
18
|
-
#
|
|
19
|
-
# @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
|
|
20
|
-
# @param minimum_selector_frequency [Integer] minimum frequency for candidate groups
|
|
21
|
-
# @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring cluster
|
|
22
|
-
def call(parsed_body, minimum_selector_frequency:)
|
|
23
|
-
new(parsed_body, minimum_frequency: minimum_selector_frequency).call
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
# @param parsed_body [Nokogiri::HTML::Document]
|
|
28
|
-
# @param minimum_frequency [Integer]
|
|
29
|
-
def initialize(parsed_body, minimum_frequency:)
|
|
30
|
-
@parsed_body = parsed_body
|
|
31
|
-
@minimum_frequency = minimum_frequency
|
|
32
|
-
@cache = {}.compare_by_identity
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
# @return [Array<Nokogiri::XML::Node>]
|
|
36
|
-
def call
|
|
37
|
-
class_result = cluster_by_class
|
|
38
|
-
return class_result unless class_result.empty?
|
|
39
|
-
|
|
40
|
-
cluster_by_structure
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
private
|
|
44
|
-
|
|
45
|
-
attr_reader :parsed_body, :minimum_frequency, :cache
|
|
46
|
-
|
|
47
|
-
def cluster_by_class
|
|
48
|
-
class_groups = collect_class_groups
|
|
49
|
-
return [] if class_groups.empty?
|
|
50
|
-
|
|
51
|
-
score_groups(class_groups)
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def cluster_by_structure
|
|
55
|
-
structure_groups = collect_structure_groups
|
|
56
|
-
return [] if structure_groups.empty?
|
|
57
|
-
|
|
58
|
-
score_groups(structure_groups)
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
def score_groups(candidate_groups)
|
|
62
|
-
scorer = GroupScorer.new
|
|
63
|
-
resolver = OverlapResolver.new(layout_tags: LAYOUT_TAG_NAMES, word_counter: scorer)
|
|
64
|
-
|
|
65
|
-
non_containers = resolver.filter_containers(candidate_groups)
|
|
66
|
-
final_groups = resolver.filter_1_to_1_overlap(non_containers)
|
|
67
|
-
|
|
68
|
-
scorer.select_best_group(final_groups)
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def collect_class_groups
|
|
72
|
-
groups = Hash.new { |h, k| h[k] = [] }
|
|
73
|
-
parsed_body.css('[class]').each { |node| add_class_node(node, groups) }
|
|
74
|
-
groups.select { |_, nodes| nodes.size >= minimum_frequency }
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def add_class_node(node, groups)
|
|
78
|
-
return if EXCLUDED_TAGS.include?(node.name)
|
|
79
|
-
return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
|
|
80
|
-
|
|
81
|
-
cls = normalize_class(node['class'])
|
|
82
|
-
groups[cls] << node unless cls.empty?
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def collect_structure_groups
|
|
86
|
-
groups = Hash.new { |h, k| h[k] = [] }
|
|
87
|
-
parsed_body.xpath('//*').each { |node| add_structure_node(node, groups) }
|
|
88
|
-
groups.select { |_, nodes| nodes.size >= minimum_frequency }
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
def add_structure_node(node, groups)
|
|
92
|
-
return if EXCLUDED_TAGS.include?(node.name)
|
|
93
|
-
return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
|
|
94
|
-
|
|
95
|
-
sig = structure_signature(node)
|
|
96
|
-
groups[sig] << node unless sig.empty?
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
def normalize_class(class_attr)
|
|
100
|
-
class_str = class_attr.to_s.strip
|
|
101
|
-
return '' if class_str.empty?
|
|
102
|
-
|
|
103
|
-
if class_str.include?(' ')
|
|
104
|
-
class_str.split(/\s+/).sort.join(' ')
|
|
105
|
-
else
|
|
106
|
-
class_str
|
|
107
|
-
end
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
def structure_signature(node)
|
|
111
|
-
child_elements = node.element_children
|
|
112
|
-
return '' if child_elements.empty?
|
|
113
|
-
|
|
114
|
-
child_elements.map(&:name).join('>')
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
end
|
|
118
|
-
end
|
|
119
|
-
end
|