html2rss 0.23.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/lib/html2rss/article.rb +42 -35
  4. data/lib/html2rss/auto_source/scraper/html.rb +68 -145
  5. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +116 -0
  6. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +63 -0
  7. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +179 -0
  8. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +55 -0
  9. data/lib/html2rss/auto_source/scraper/json_state.rb +0 -379
  10. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +29 -114
  11. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +50 -11
  12. data/lib/html2rss/auto_source/scraper/schema/thing.rb +14 -5
  13. data/lib/html2rss/auto_source/scraper/schema.rb +61 -18
  14. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +35 -30
  15. data/lib/html2rss/auto_source/scraper/semantic_html.rb +40 -120
  16. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +161 -0
  17. data/lib/html2rss/auto_source/scraper/sitemap.rb +10 -7
  18. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +6 -2
  19. data/lib/html2rss/auto_source/scraper.rb +81 -44
  20. data/lib/html2rss/auto_source/segment.rb +29 -0
  21. data/lib/html2rss/auto_source/segmenter/cluster.rb +163 -0
  22. data/lib/html2rss/auto_source/segmenter/list.rb +101 -0
  23. data/lib/html2rss/auto_source/segmenter/primary_link.rb +81 -0
  24. data/lib/html2rss/auto_source/segmenter/semantic.rb +104 -0
  25. data/lib/html2rss/auto_source/segmenter.rb +66 -0
  26. data/lib/html2rss/auto_source.rb +74 -21
  27. data/lib/html2rss/channel.rb +159 -79
  28. data/lib/html2rss/cli.rb +8 -13
  29. data/lib/html2rss/config/auto_source_contract.rb +2 -0
  30. data/lib/html2rss/config/request_controls.rb +33 -0
  31. data/lib/html2rss/config/validator.rb +15 -8
  32. data/lib/html2rss/config.rb +6 -2
  33. data/lib/html2rss/feed_builder/item_presentation.rb +42 -0
  34. data/lib/html2rss/feed_builder/json_feed/item.rb +20 -4
  35. data/lib/html2rss/feed_builder/json_feed.rb +22 -2
  36. data/lib/html2rss/feed_builder/rss.rb +36 -24
  37. data/lib/html2rss/feed_builder.rb +9 -17
  38. data/lib/html2rss/feed_pipeline/auto_fallback.rb +71 -28
  39. data/lib/html2rss/feed_pipeline.rb +66 -55
  40. data/lib/html2rss/feed_result.rb +92 -0
  41. data/lib/html2rss/html/article_extractor/category_extractor.rb +10 -36
  42. data/lib/html2rss/html/article_extractor/date_extractor.rb +2 -7
  43. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +5 -50
  44. data/lib/html2rss/html/article_extractor/image_extractor.rb +5 -16
  45. data/lib/html2rss/html/article_rules/category.rb +55 -0
  46. data/lib/html2rss/html/article_rules/date.rb +25 -0
  47. data/lib/html2rss/html/article_rules/enclosure.rb +72 -0
  48. data/lib/html2rss/html/article_rules/image.rb +109 -0
  49. data/lib/html2rss/html/article_rules.rb +10 -0
  50. data/lib/html2rss/html/rendering/audio_renderer.rb +2 -12
  51. data/lib/html2rss/html/rendering/escaped_attributes.rb +27 -0
  52. data/lib/html2rss/html/rendering/image_renderer.rb +2 -12
  53. data/lib/html2rss/html/rendering/pdf_renderer.rb +2 -8
  54. data/lib/html2rss/html/rendering/video_renderer.rb +2 -12
  55. data/lib/html2rss/html/sst_article_extractor.rb +279 -0
  56. data/lib/html2rss/link_destination/destination_facts.rb +40 -0
  57. data/lib/html2rss/link_destination/noise_policy.rb +79 -0
  58. data/lib/html2rss/link_destination/path_classifier.rb +205 -0
  59. data/lib/html2rss/link_destination/text_classifier.rb +64 -0
  60. data/lib/html2rss/link_destination.rb +8 -0
  61. data/lib/html2rss/request_service/botasaurus_contract.rb +33 -5
  62. data/lib/html2rss/request_service/botasaurus_strategy.rb +13 -4
  63. data/lib/html2rss/request_service/budget.rb +7 -2
  64. data/lib/html2rss/request_service/faraday_strategy.rb +42 -1
  65. data/lib/html2rss/request_service/network_guard.rb +5 -3
  66. data/lib/html2rss/request_service/response.rb +9 -1
  67. data/lib/html2rss/request_service/strategy.rb +22 -0
  68. data/lib/html2rss/request_session.rb +4 -0
  69. data/lib/html2rss/scoring/anchor_score.rb +34 -0
  70. data/lib/html2rss/scoring/cluster_scorer.rb +87 -0
  71. data/lib/html2rss/scoring/container_assessor.rb +83 -0
  72. data/lib/html2rss/scoring/engine.rb +101 -0
  73. data/lib/html2rss/scoring/link_resolver.rb +72 -0
  74. data/lib/html2rss/scoring/observation.rb +53 -0
  75. data/lib/html2rss/scoring/ranked_segment.rb +45 -0
  76. data/lib/html2rss/scoring/score.rb +30 -0
  77. data/lib/html2rss/scoring.rb +33 -0
  78. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +10 -1
  79. data/lib/html2rss/selectors.rb +26 -24
  80. data/lib/html2rss/sst/attrs.rb +91 -0
  81. data/lib/html2rss/sst/document.rb +23 -0
  82. data/lib/html2rss/sst/index.rb +112 -0
  83. data/lib/html2rss/sst/node.rb +147 -0
  84. data/lib/html2rss/sst/normalizer.rb +171 -0
  85. data/lib/html2rss/sst/tags.rb +26 -0
  86. data/lib/html2rss/sst/text.rb +81 -0
  87. data/lib/html2rss/sst.rb +8 -0
  88. data/lib/html2rss/status.rb +132 -0
  89. data/lib/html2rss/version.rb +1 -1
  90. data/lib/html2rss.rb +41 -29
  91. data/schema/html2rss-config.schema.json +20 -0
  92. metadata +45 -19
  93. data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +0 -80
  94. data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +0 -86
  95. data/lib/html2rss/auto_source/discovery/dom_clustering.rb +0 -119
  96. data/lib/html2rss/auto_source/discovery/list_candidates.rb +0 -94
  97. data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +0 -219
  98. data/lib/html2rss/auto_source/discovery/semantic_containers.rb +0 -71
  99. data/lib/html2rss/auto_source/discovery/sitemap.rb +0 -159
  100. data/lib/html2rss/auto_source/discovery.rb +0 -14
  101. data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +0 -28
  102. data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +0 -106
  103. data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +0 -80
  104. data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +0 -42
  105. data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +0 -38
  106. data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +0 -221
  107. data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +0 -66
  108. data/lib/html2rss/auto_source/link_heuristics.rb +0 -139
  109. data/lib/html2rss/auto_source/scraper/schema/list_item.rb +0 -28
data/lib/html2rss.rb CHANGED
@@ -3,7 +3,10 @@
3
3
  require 'zeitwerk'
4
4
 
5
5
  loader = Zeitwerk::Loader.for_gem
6
- loader.inflector.inflect('cli' => 'CLI')
6
+ loader.inflector.inflect(
7
+ 'cli' => 'CLI',
8
+ 'sst' => 'SST'
9
+ )
7
10
  loader.setup
8
11
 
9
12
  require 'logger'
@@ -34,28 +37,43 @@ module Html2rss
34
37
  Config.load_yaml(file, feed_name)
35
38
  end
36
39
 
40
+ ##
41
+ # Returns an opaque, Marshal-cacheable result of one scrape.
42
+ #
43
+ # Prefer this when the same scrape must render as RSS and JSON Feed (e.g. web cache).
44
+ #
45
+ # @param raw_config [Hash{Symbol => Object}] feed configuration
46
+ # @return [Html2rss::FeedResult]
47
+ def self.feed_result(raw_config)
48
+ FeedPipeline.new(raw_config).to_result
49
+ end
50
+
37
51
  ##
38
52
  # Returns an RSS object generated from the provided configuration.
39
53
  #
40
54
  # @param raw_config [Hash{Symbol => Object}] feed configuration
41
55
  # @return [RSS::Rss] generated RSS feed
42
56
  def self.feed(raw_config)
43
- FeedPipeline.new(raw_config).to_rss
57
+ feed_result(raw_config).to_rss
44
58
  end
45
59
 
46
60
  ##
47
61
  # Returns a JSONFeed 1.1 hash generated from the provided configuration.
48
62
  #
49
63
  # @param raw_config [Hash{Symbol => Object}] feed configuration
64
+ # @param feed_url [String, nil] optional self URL for the feed (JSON Feed +feed_url+)
50
65
  # @return [Hash] JSONFeed-compliant hash
51
- def self.json_feed(raw_config)
52
- FeedPipeline.new(raw_config).to_json_feed
66
+ def self.json_feed(raw_config, feed_url: nil)
67
+ feed_result(raw_config).to_json_feed(feed_url:)
53
68
  end
54
69
 
55
70
  # rubocop:disable Metrics/ParameterLists
56
71
 
57
72
  ##
58
- # Scrapes the provided URL and returns an RSS object.
73
+ # Scrapes the provided URL without hand-written selectors and returns an RSS object.
74
+ #
75
+ # Builds an auto_source config, then FeedPipeline runs structured scrapers and
76
+ # (when needed) the SST heuristic path — see docs/auto_source.md.
59
77
  #
60
78
  # @param url [String] source page URL
61
79
  # @param strategy [Symbol] request strategy to use
@@ -63,18 +81,23 @@ module Html2rss
63
81
  # @param max_redirects [Integer, nil] optional redirect limit override
64
82
  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
65
83
  # @param local_file_path [String, nil] optional local HTML file path
84
+ # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
66
85
  # @return [RSS::Rss] generated RSS feed
67
86
  def self.auto_source(url,
68
87
  strategy: :auto,
69
88
  items_selector: nil,
70
89
  max_redirects: nil,
71
90
  max_requests: 4,
72
- local_file_path: nil)
73
- feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
91
+ local_file_path: nil,
92
+ limit: nil)
93
+ feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
94
+ local_file_path:, limit:))
74
95
  end
75
96
 
76
97
  ##
77
- # Scrapes the provided URL and returns a JSONFeed 1.1 hash.
98
+ # Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
99
+ #
100
+ # Same auto_source pipeline as {.auto_source}; see docs/auto_source.md.
78
101
  #
79
102
  # @param url [String] source page URL
80
103
  # @param strategy [Symbol] request strategy to use
@@ -82,15 +105,17 @@ module Html2rss
82
105
  # @param max_redirects [Integer, nil] optional redirect limit override
83
106
  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
84
107
  # @param local_file_path [String, nil] optional local HTML file path
108
+ # @param limit [Integer, nil] max articles to keep (default: {AutoSource::DEFAULT_LIMIT})
85
109
  # @return [Hash] JSONFeed-compliant hash
86
110
  def self.auto_json_feed(url,
87
111
  strategy: :auto,
88
112
  items_selector: nil,
89
113
  max_redirects: nil,
90
114
  max_requests: 4,
91
- local_file_path: nil)
115
+ local_file_path: nil,
116
+ limit: nil)
92
117
  json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
93
- local_file_path:))
118
+ local_file_path:, limit:))
94
119
  end
95
120
 
96
121
  # rubocop:enable Metrics/ParameterLists
@@ -142,11 +167,14 @@ module Html2rss
142
167
  class << self
143
168
  private
144
169
 
145
- def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path: nil) # rubocop:disable Metrics/ParameterLists
170
+ # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
171
+ def build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
172
+ local_file_path: nil, limit: nil)
146
173
  config = Config.auto_source_config(
147
174
  url:,
148
175
  items_selector:,
149
- request_controls: shortcut_request_controls(strategy:, max_redirects:, max_requests:)
176
+ request_controls: Config::RequestControls.from_shortcut(strategy:, max_redirects:, max_requests:),
177
+ limit:
150
178
  )
151
179
  if local_file_path
152
180
  config[:request] ||= {}
@@ -154,23 +182,7 @@ module Html2rss
154
182
  end
155
183
  config
156
184
  end
157
-
158
- def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
159
- Config::RequestControls.new(
160
- strategy:,
161
- max_redirects:,
162
- max_requests:,
163
- explicit_keys: explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
164
- )
165
- end
166
-
167
- def explicit_request_control_keys(strategy:, max_redirects:, max_requests:)
168
- keys = []
169
- keys << :strategy unless strategy.nil? || strategy == Config.default_strategy_name
170
- keys << :max_redirects unless max_redirects.nil?
171
- keys << :max_requests unless max_requests.nil?
172
- keys
173
- end
185
+ # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
174
186
  end
175
187
 
176
188
  logger.level = defaults.log_level if logger.respond_to?(:level=)
@@ -45,6 +45,18 @@
45
45
  "null",
46
46
  "string"
47
47
  ]
48
+ },
49
+ "author": {
50
+ "type": [
51
+ "null",
52
+ "string"
53
+ ]
54
+ },
55
+ "image": {
56
+ "type": [
57
+ "null",
58
+ "string"
59
+ ]
48
60
  }
49
61
  },
50
62
  "required": [
@@ -94,6 +106,13 @@
94
106
  "$schema": "http://json-schema.org/draft-06/schema#",
95
107
  "type": "object",
96
108
  "properties": {
109
+ "limit": {
110
+ "type": "integer",
111
+ "not": {
112
+ "type": "null"
113
+ },
114
+ "exclusiveMinimum": 0
115
+ },
97
116
  "scraper": {
98
117
  "type": "object",
99
118
  "properties": {
@@ -269,6 +288,7 @@
269
288
  },
270
289
  "required": [],
271
290
  "default": {
291
+ "limit": 25,
272
292
  "scraper": {
273
293
  "wordpress_api": {
274
294
  "enabled": true
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: html2rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.23.0
4
+ version: 0.25.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gil Desmarais
@@ -278,40 +278,34 @@ files:
278
278
  - lib/html2rss/article/enclosure.rb
279
279
  - lib/html2rss/auto_source.rb
280
280
  - lib/html2rss/auto_source/cleanup.rb
281
- - lib/html2rss/auto_source/discovery.rb
282
- - lib/html2rss/auto_source/discovery/dom_clustering.rb
283
- - lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb
284
- - lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb
285
- - lib/html2rss/auto_source/discovery/list_candidates.rb
286
- - lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb
287
- - lib/html2rss/auto_source/discovery/semantic_containers.rb
288
- - lib/html2rss/auto_source/discovery/sitemap.rb
289
- - lib/html2rss/auto_source/link_heuristics.rb
290
- - lib/html2rss/auto_source/link_heuristics/anchor_signals.rb
291
- - lib/html2rss/auto_source/link_heuristics/container_assessor.rb
292
- - lib/html2rss/auto_source/link_heuristics/container_signals.rb
293
- - lib/html2rss/auto_source/link_heuristics/destination_facts.rb
294
- - lib/html2rss/auto_source/link_heuristics/href_extractor.rb
295
- - lib/html2rss/auto_source/link_heuristics/path_classifier.rb
296
- - lib/html2rss/auto_source/link_heuristics/text_classifier.rb
297
281
  - lib/html2rss/auto_source/scraper.rb
298
282
  - lib/html2rss/auto_source/scraper/html.rb
299
283
  - lib/html2rss/auto_source/scraper/json_state.rb
284
+ - lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb
285
+ - lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb
286
+ - lib/html2rss/auto_source/scraper/json_state/document_scanner.rb
287
+ - lib/html2rss/auto_source/scraper/json_state/value_finder.rb
300
288
  - lib/html2rss/auto_source/scraper/meta_oembed.rb
301
289
  - lib/html2rss/auto_source/scraper/microdata.rb
302
290
  - lib/html2rss/auto_source/scraper/microformats2.rb
303
291
  - lib/html2rss/auto_source/scraper/schema.rb
304
292
  - lib/html2rss/auto_source/scraper/schema/category_extractor.rb
305
293
  - lib/html2rss/auto_source/scraper/schema/item_list.rb
306
- - lib/html2rss/auto_source/scraper/schema/list_item.rb
307
294
  - lib/html2rss/auto_source/scraper/schema/thing.rb
308
295
  - lib/html2rss/auto_source/scraper/semantic_html.rb
309
296
  - lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb
310
297
  - lib/html2rss/auto_source/scraper/sitemap.rb
298
+ - lib/html2rss/auto_source/scraper/sitemap/parser.rb
311
299
  - lib/html2rss/auto_source/scraper/wordpress_api.rb
312
300
  - lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb
313
301
  - lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb
314
302
  - lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb
303
+ - lib/html2rss/auto_source/segment.rb
304
+ - lib/html2rss/auto_source/segmenter.rb
305
+ - lib/html2rss/auto_source/segmenter/cluster.rb
306
+ - lib/html2rss/auto_source/segmenter/list.rb
307
+ - lib/html2rss/auto_source/segmenter/primary_link.rb
308
+ - lib/html2rss/auto_source/segmenter/semantic.rb
315
309
  - lib/html2rss/channel.rb
316
310
  - lib/html2rss/cli.rb
317
311
  - lib/html2rss/config.rb
@@ -326,6 +320,7 @@ files:
326
320
  - lib/html2rss/defaults.rb
327
321
  - lib/html2rss/error.rb
328
322
  - lib/html2rss/feed_builder.rb
323
+ - lib/html2rss/feed_builder/item_presentation.rb
329
324
  - lib/html2rss/feed_builder/json_feed.rb
330
325
  - lib/html2rss/feed_builder/json_feed/item.rb
331
326
  - lib/html2rss/feed_builder/rss.rb
@@ -334,6 +329,7 @@ files:
334
329
  - lib/html2rss/feed_pipeline/auto_fallback.rb
335
330
  - lib/html2rss/feed_pipeline/runtime_policy.rb
336
331
  - lib/html2rss/feed_pipeline/strategy_plan.rb
332
+ - lib/html2rss/feed_result.rb
337
333
  - lib/html2rss/hash_util.rb
338
334
  - lib/html2rss/html.rb
339
335
  - lib/html2rss/html/article_extractor.rb
@@ -343,15 +339,27 @@ files:
343
339
  - lib/html2rss/html/article_extractor/heading_extractor.rb
344
340
  - lib/html2rss/html/article_extractor/id_generator.rb
345
341
  - lib/html2rss/html/article_extractor/image_extractor.rb
342
+ - lib/html2rss/html/article_rules.rb
343
+ - lib/html2rss/html/article_rules/category.rb
344
+ - lib/html2rss/html/article_rules/date.rb
345
+ - lib/html2rss/html/article_rules/enclosure.rb
346
+ - lib/html2rss/html/article_rules/image.rb
346
347
  - lib/html2rss/html/navigator.rb
347
348
  - lib/html2rss/html/navigator/text_extractor.rb
348
349
  - lib/html2rss/html/rendering.rb
349
350
  - lib/html2rss/html/rendering/audio_renderer.rb
350
351
  - lib/html2rss/html/rendering/description_builder.rb
352
+ - lib/html2rss/html/rendering/escaped_attributes.rb
351
353
  - lib/html2rss/html/rendering/image_renderer.rb
352
354
  - lib/html2rss/html/rendering/media_renderer.rb
353
355
  - lib/html2rss/html/rendering/pdf_renderer.rb
354
356
  - lib/html2rss/html/rendering/video_renderer.rb
357
+ - lib/html2rss/html/sst_article_extractor.rb
358
+ - lib/html2rss/link_destination.rb
359
+ - lib/html2rss/link_destination/destination_facts.rb
360
+ - lib/html2rss/link_destination/noise_policy.rb
361
+ - lib/html2rss/link_destination/path_classifier.rb
362
+ - lib/html2rss/link_destination/text_classifier.rb
355
363
  - lib/html2rss/request_service.rb
356
364
  - lib/html2rss/request_service/blocked_surface.rb
357
365
  - lib/html2rss/request_service/botasaurus_contract.rb
@@ -377,6 +385,15 @@ files:
377
385
  - lib/html2rss/request_session/pager/offset.rb
378
386
  - lib/html2rss/request_session/pager/rel_next.rb
379
387
  - lib/html2rss/request_session/pager/url_template.rb
388
+ - lib/html2rss/scoring.rb
389
+ - lib/html2rss/scoring/anchor_score.rb
390
+ - lib/html2rss/scoring/cluster_scorer.rb
391
+ - lib/html2rss/scoring/container_assessor.rb
392
+ - lib/html2rss/scoring/engine.rb
393
+ - lib/html2rss/scoring/link_resolver.rb
394
+ - lib/html2rss/scoring/observation.rb
395
+ - lib/html2rss/scoring/ranked_segment.rb
396
+ - lib/html2rss/scoring/score.rb
380
397
  - lib/html2rss/selectors.rb
381
398
  - lib/html2rss/selectors/extractors.rb
382
399
  - lib/html2rss/selectors/extractors/attribute.rb
@@ -398,6 +415,15 @@ files:
398
415
  - lib/html2rss/selectors/post_processors/sanitize_html.rb
399
416
  - lib/html2rss/selectors/post_processors/substring.rb
400
417
  - lib/html2rss/selectors/post_processors/template.rb
418
+ - lib/html2rss/sst.rb
419
+ - lib/html2rss/sst/attrs.rb
420
+ - lib/html2rss/sst/document.rb
421
+ - lib/html2rss/sst/index.rb
422
+ - lib/html2rss/sst/node.rb
423
+ - lib/html2rss/sst/normalizer.rb
424
+ - lib/html2rss/sst/tags.rb
425
+ - lib/html2rss/sst/text.rb
426
+ - lib/html2rss/status.rb
401
427
  - lib/html2rss/url.rb
402
428
  - lib/html2rss/version.rb
403
429
  - lib/tasks/config_schema.rake
@@ -407,7 +433,7 @@ licenses:
407
433
  - MIT
408
434
  metadata:
409
435
  allowed_push_host: https://rubygems.org
410
- changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.23.0
436
+ changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.25.0
411
437
  rubygems_mfa_required: 'true'
412
438
  rdoc_options: []
413
439
  require_paths:
@@ -1,80 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Html2rss
4
- class AutoSource
5
- module Discovery
6
- class DomClustering
7
- ##
8
- # Scores candidate DOM groups using heading, time, date, and word-count signals.
9
- class GroupScorer
10
- def initialize
11
- @text_words = {}.compare_by_identity
12
- @has_date = {}.compare_by_identity
13
- end
14
-
15
- # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate DOM groups
16
- # @return [Array<Nokogiri::XML::Node>] nodes from the highest-scoring group
17
- def select_best_group(groups)
18
- best_nodes = []
19
- best_score = -1
20
-
21
- groups.each_value do |nodes|
22
- score = score_group(nodes)
23
- next if score.negative?
24
-
25
- (best_nodes = nodes) && (best_score = score) if score > best_score
26
- end
27
-
28
- best_nodes
29
- end
30
-
31
- # @param nodes [Array<Nokogiri::XML::Node>]
32
- # @return [Float] average visible word count across nodes
33
- def avg_words(nodes)
34
- nodes.sum { |n| text_words(n) } / nodes.size.to_f
35
- end
36
-
37
- private
38
-
39
- def score_group(nodes)
40
- avg_w = avg_words(nodes)
41
- return -1 if avg_w < 5
42
-
43
- score = nodes.size + (avg_w / 5.0)
44
- score += 20 if nodes_heading?(nodes)
45
- score += 20 if nodes_time?(nodes)
46
- score += 40 if nodes_date?(nodes)
47
- score
48
- end
49
-
50
- def nodes_heading?(nodes)
51
- nodes.any? do |n|
52
- n.at_css(Html2rss::Html::Navigator::HEADING_TAGS.join(',')) ||
53
- n.at_css('.font-bold, .font-semibold')
54
- end
55
- end
56
-
57
- def nodes_time?(nodes)
58
- nodes.any? { |n| n.at_css('time, [datetime]') }
59
- end
60
-
61
- def nodes_date?(nodes)
62
- nodes.any? { |n| date?(n) }
63
- end
64
-
65
- def text_words(node)
66
- @text_words[node] ||= Html2rss::Html::Navigator.extract_visible_text(node).to_s.scan(/\p{Alnum}+/).size
67
- end
68
-
69
- def date?(node)
70
- @has_date[node] ||= begin
71
- text = Html2rss::Html::Navigator.extract_visible_text(node).to_s
72
- text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
73
- text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
74
- end
75
- end
76
- end
77
- end
78
- end
79
- end
80
- end
@@ -1,86 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Html2rss
4
- class AutoSource
5
- module Discovery
6
- class DomClustering
7
- ##
8
- # Filters layout wrapper groups and resolves 1-to-1 nested card wrappers.
9
- class OverlapResolver
10
- # @param layout_tags [Set<String>] tags treated as layout containers
11
- # @param word_counter [#avg_words] averages visible word counts for nodes
12
- def initialize(layout_tags:, word_counter:)
13
- @layout_tags = layout_tags
14
- @word_counter = word_counter
15
- end
16
-
17
- # Discard group A if any node of A contains > 1 node of another group B
18
- #
19
- # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate groups
20
- # @return [Hash{String => Array<Nokogiri::XML::Node>}] groups that are not layout wrappers
21
- def filter_containers(groups)
22
- groups.reject do |cls_a, nodes_a|
23
- groups.any? { |cls_b, nodes_b| cls_a != cls_b && container_of?(nodes_a, nodes_b) }
24
- end
25
- end
26
-
27
- # If group A contains group B with the same size, keep the real content card.
28
- #
29
- # @param groups [Hash{String => Array<Nokogiri::XML::Node>}] candidate groups
30
- # @return [Hash{String => Array<Nokogiri::XML::Node>}] groups after 1-to-1 resolution
31
- def filter_1_to_1_overlap(groups)
32
- discarded = Set.new
33
- groups.each_key do |cls_a|
34
- groups.each_key do |cls_b|
35
- next if cls_a == cls_b || discarded.include?(cls_a) || discarded.include?(cls_b)
36
-
37
- resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
38
- end
39
- end
40
-
41
- groups.except(*discarded)
42
- end
43
-
44
- private
45
-
46
- attr_reader :layout_tags, :word_counter
47
-
48
- # rubocop:disable Metrics/MethodLength
49
- def container_of?(nodes_a, nodes_b)
50
- return false unless layout_tags.include?(nodes_b.first.name)
51
-
52
- nodes_a.any? do |node_a|
53
- count = 0
54
- nodes_b.each do |node_b|
55
- next if node_a == node_b
56
-
57
- if Html2rss::Html::Navigator.descendant_of?(node_b, node_a)
58
- count += 1
59
- break if count > 1
60
- end
61
- end
62
- count > 1
63
- end
64
- end
65
- # rubocop:enable Metrics/MethodLength
66
-
67
- def resolve_1_to_1_overlap(cls_a, cls_b, groups, discarded)
68
- nodes_a = groups[cls_a]
69
- nodes_b = groups[cls_b]
70
- return if nodes_a.size != nodes_b.size
71
-
72
- nested = nodes_a.zip(nodes_b).all? { |a, b| a != b && Html2rss::Html::Navigator.descendant_of?(b, a) }
73
- return unless nested
74
-
75
- discarded << (keep_descendant?(nodes_a, nodes_b) ? cls_a : cls_b)
76
- end
77
-
78
- def keep_descendant?(nodes_a, nodes_b)
79
- word_counter.avg_words(nodes_b) >= 0.8 * word_counter.avg_words(nodes_a) &&
80
- layout_tags.include?(nodes_b.first.name)
81
- end
82
- end
83
- end
84
- end
85
- end
86
- end
@@ -1,119 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Html2rss
4
- class AutoSource
5
- module Discovery
6
- ##
7
- # Discovers repeated content card nodes in anchorless or classless DOM trees
8
- # by evaluating class groups first and falling back to 1-level tag structure signatures.
9
- class DomClustering
10
- # Node tags considered layout containers
11
- LAYOUT_TAG_NAMES = Set['div', 'section', 'article', 'li', 'ul', 'ol'].freeze
12
- # HTML/layout tags excluded from candidate nodes (owned by Html::Navigator).
13
- EXCLUDED_TAGS = Html2rss::Html::Navigator::CLUSTER_EXCLUDED_TAGS
14
-
15
- class << self
16
- ##
17
- # Clusters elements in parsed_body and returns the best set of content card nodes.
18
- #
19
- # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
20
- # @param minimum_selector_frequency [Integer] minimum frequency for candidate groups
21
- # @return [Array<Nokogiri::XML::Node>] candidate nodes of the top-scoring cluster
22
- def call(parsed_body, minimum_selector_frequency:)
23
- new(parsed_body, minimum_frequency: minimum_selector_frequency).call
24
- end
25
- end
26
-
27
- # @param parsed_body [Nokogiri::HTML::Document]
28
- # @param minimum_frequency [Integer]
29
- def initialize(parsed_body, minimum_frequency:)
30
- @parsed_body = parsed_body
31
- @minimum_frequency = minimum_frequency
32
- @cache = {}.compare_by_identity
33
- end
34
-
35
- # @return [Array<Nokogiri::XML::Node>]
36
- def call
37
- class_result = cluster_by_class
38
- return class_result unless class_result.empty?
39
-
40
- cluster_by_structure
41
- end
42
-
43
- private
44
-
45
- attr_reader :parsed_body, :minimum_frequency, :cache
46
-
47
- def cluster_by_class
48
- class_groups = collect_class_groups
49
- return [] if class_groups.empty?
50
-
51
- score_groups(class_groups)
52
- end
53
-
54
- def cluster_by_structure
55
- structure_groups = collect_structure_groups
56
- return [] if structure_groups.empty?
57
-
58
- score_groups(structure_groups)
59
- end
60
-
61
- def score_groups(candidate_groups)
62
- scorer = GroupScorer.new
63
- resolver = OverlapResolver.new(layout_tags: LAYOUT_TAG_NAMES, word_counter: scorer)
64
-
65
- non_containers = resolver.filter_containers(candidate_groups)
66
- final_groups = resolver.filter_1_to_1_overlap(non_containers)
67
-
68
- scorer.select_best_group(final_groups)
69
- end
70
-
71
- def collect_class_groups
72
- groups = Hash.new { |h, k| h[k] = [] }
73
- parsed_body.css('[class]').each { |node| add_class_node(node, groups) }
74
- groups.select { |_, nodes| nodes.size >= minimum_frequency }
75
- end
76
-
77
- def add_class_node(node, groups)
78
- return if EXCLUDED_TAGS.include?(node.name)
79
- return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
80
-
81
- cls = normalize_class(node['class'])
82
- groups[cls] << node unless cls.empty?
83
- end
84
-
85
- def collect_structure_groups
86
- groups = Hash.new { |h, k| h[k] = [] }
87
- parsed_body.xpath('//*').each { |node| add_structure_node(node, groups) }
88
- groups.select { |_, nodes| nodes.size >= minimum_frequency }
89
- end
90
-
91
- def add_structure_node(node, groups)
92
- return if EXCLUDED_TAGS.include?(node.name)
93
- return if Html2rss::Html::Navigator.ignored_container_path?(node, cache)
94
-
95
- sig = structure_signature(node)
96
- groups[sig] << node unless sig.empty?
97
- end
98
-
99
- def normalize_class(class_attr)
100
- class_str = class_attr.to_s.strip
101
- return '' if class_str.empty?
102
-
103
- if class_str.include?(' ')
104
- class_str.split(/\s+/).sort.join(' ')
105
- else
106
- class_str
107
- end
108
- end
109
-
110
- def structure_signature(node)
111
- child_elements = node.element_children
112
- return '' if child_elements.empty?
113
-
114
- child_elements.map(&:name).join('>')
115
- end
116
- end
117
- end
118
- end
119
- end