html2rss 0.33.0 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -2
  3. data/lib/html2rss/article/deduplicator.rb +5 -2
  4. data/lib/html2rss/article/enclosure.rb +185 -21
  5. data/lib/html2rss/article/title.rb +203 -0
  6. data/lib/html2rss/article.rb +79 -13
  7. data/lib/html2rss/auto_source/README.md +16 -8
  8. data/lib/html2rss/auto_source/cleanup.rb +126 -50
  9. data/lib/html2rss/auto_source/corroboration.rb +167 -0
  10. data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
  11. data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
  12. data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
  13. data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
  14. data/lib/html2rss/auto_source/page_links.rb +164 -0
  15. data/lib/html2rss/auto_source/scraper/html.rb +11 -39
  16. data/lib/html2rss/auto_source/scraper/json_document.rb +1 -3
  17. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +119 -42
  18. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +5 -16
  19. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +2 -10
  20. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -3
  21. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +6 -14
  22. data/lib/html2rss/auto_source/scraper/microdata.rb +6 -4
  23. data/lib/html2rss/auto_source/scraper/microformats2.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/native_feed.rb +27 -17
  25. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
  26. data/lib/html2rss/auto_source/scraper/schema/thing.rb +68 -27
  27. data/lib/html2rss/auto_source/scraper/schema.rb +5 -8
  28. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
  29. data/lib/html2rss/auto_source/scraper/semantic_html.rb +1 -5
  30. data/lib/html2rss/auto_source/scraper/sitemap.rb +4 -3
  31. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -3
  32. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -3
  33. data/lib/html2rss/auto_source/scraper.rb +10 -2
  34. data/lib/html2rss/auto_source/segment.rb +19 -5
  35. data/lib/html2rss/auto_source/segmenter/cluster.rb +13 -11
  36. data/lib/html2rss/auto_source/segmenter/list.rb +13 -5
  37. data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
  38. data/lib/html2rss/auto_source/segmenter/semantic.rb +17 -6
  39. data/lib/html2rss/auto_source/segmenter.rb +4 -8
  40. data/lib/html2rss/auto_source.rb +81 -26
  41. data/lib/html2rss/batch.rb +2 -0
  42. data/lib/html2rss/capture/README.md +5 -2
  43. data/lib/html2rss/capture.rb +44 -32
  44. data/lib/html2rss/channel.rb +30 -12
  45. data/lib/html2rss/cli.rb +22 -11
  46. data/lib/html2rss/config/auto_source_contract.rb +52 -45
  47. data/lib/html2rss/config/issue_mapper.rb +4 -1
  48. data/lib/html2rss/config/request_controls.rb +6 -3
  49. data/lib/html2rss/config/schema.rb +15 -5
  50. data/lib/html2rss/config/selectors_validator.rb +2 -2
  51. data/lib/html2rss/config/validation_issue.rb +4 -1
  52. data/lib/html2rss/config/validator.rb +11 -0
  53. data/lib/html2rss/config.rb +53 -27
  54. data/lib/html2rss/contract_error.rb +9 -0
  55. data/lib/html2rss/defaults.rb +21 -23
  56. data/lib/html2rss/doctor/botasaurus.rb +7 -3
  57. data/lib/html2rss/error.rb +10 -0
  58. data/lib/html2rss/error_kind.rb +39 -0
  59. data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
  60. data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
  61. data/lib/html2rss/feed_builder/json_feed.rb +2 -2
  62. data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
  63. data/lib/html2rss/feed_builder/rss.rb +49 -6
  64. data/lib/html2rss/feed_builder.rb +2 -1
  65. data/lib/html2rss/feed_pipeline/README.md +1 -1
  66. data/lib/html2rss/feed_pipeline/auto_fallback.rb +13 -31
  67. data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
  68. data/lib/html2rss/feed_pipeline.rb +40 -27
  69. data/lib/html2rss/feed_resolution/README.md +1 -1
  70. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  71. data/lib/html2rss/feed_resolution/policy.rb +1 -1
  72. data/lib/html2rss/feed_resolution/probe.rb +3 -1
  73. data/lib/html2rss/feed_resolution.rb +11 -12
  74. data/lib/html2rss/feed_result.rb +9 -5
  75. data/lib/html2rss/hash_util.rb +3 -3
  76. data/lib/html2rss/html/article_extractor/category_extractor.rb +1 -1
  77. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -31
  78. data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
  79. data/lib/html2rss/html/article_extractor/id_generator.rb +6 -9
  80. data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
  81. data/lib/html2rss/html/article_extractor.rb +19 -47
  82. data/lib/html2rss/html/article_rules/category.rb +3 -3
  83. data/lib/html2rss/html/article_rules/description.rb +6 -15
  84. data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
  85. data/lib/html2rss/html/article_rules/image.rb +30 -16
  86. data/lib/html2rss/html/feed_link.rb +7 -9
  87. data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
  88. data/lib/html2rss/html/navigator.rb +39 -9
  89. data/lib/html2rss/html/probe.rb +4 -0
  90. data/lib/html2rss/html/rendering/description_builder.rb +46 -32
  91. data/lib/html2rss/html/rendering.rb +2 -9
  92. data/lib/html2rss/html/sst_article_extractor.rb +55 -145
  93. data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
  94. data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
  95. data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
  96. data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
  97. data/lib/html2rss/html/title_pieces.rb +169 -0
  98. data/lib/html2rss/html/visibility.rb +56 -0
  99. data/lib/html2rss/html.rb +2 -0
  100. data/lib/html2rss/link_destination/destination_facts.rb +17 -1
  101. data/lib/html2rss/link_destination/noise_policy.rb +11 -10
  102. data/lib/html2rss/link_destination/path_classifier.rb +100 -9
  103. data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
  104. data/lib/html2rss/link_destination/sponsorship.rb +184 -0
  105. data/lib/html2rss/link_destination/text_classifier.rb +104 -8
  106. data/lib/html2rss/mcp/README.md +13 -1
  107. data/lib/html2rss/mcp/config_argument.rb +5 -2
  108. data/lib/html2rss/mcp/contract.rb +13 -4
  109. data/lib/html2rss/mcp/outcome.rb +14 -12
  110. data/lib/html2rss/mcp/runtime.rb +39 -2
  111. data/lib/html2rss/mcp/server.rb +13 -4
  112. data/lib/html2rss/page_recon/README.md +1 -1
  113. data/lib/html2rss/page_recon/diagnostics.rb +2 -0
  114. data/lib/html2rss/recon.rb +10 -12
  115. data/lib/html2rss/request_service/blocked_surface.rb +2 -1
  116. data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
  117. data/lib/html2rss/request_service/botasaurus_strategy.rb +42 -10
  118. data/lib/html2rss/request_service/context.rb +15 -14
  119. data/lib/html2rss/request_service/cookie_jar.rb +89 -0
  120. data/lib/html2rss/request_service/cookie_store.rb +79 -0
  121. data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
  122. data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
  123. data/lib/html2rss/request_service/local_file_strategy.rb +2 -2
  124. data/lib/html2rss/request_service/policy.rb +48 -11
  125. data/lib/html2rss/request_service/replay_breaker.rb +56 -0
  126. data/lib/html2rss/request_service/response.rb +30 -13
  127. data/lib/html2rss/request_service/response_cache.rb +109 -0
  128. data/lib/html2rss/request_service/response_caching.rb +95 -0
  129. data/lib/html2rss/request_service/strategy.rb +5 -11
  130. data/lib/html2rss/request_service.rb +71 -12
  131. data/lib/html2rss/request_session/pager/base.rb +0 -7
  132. data/lib/html2rss/request_session/pager.rb +1 -1
  133. data/lib/html2rss/request_session.rb +6 -2
  134. data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
  135. data/lib/html2rss/scoring/container_assessor.rb +12 -13
  136. data/lib/html2rss/scoring/engine.rb +6 -6
  137. data/lib/html2rss/scoring/observation.rb +2 -2
  138. data/lib/html2rss/selector_candidates.rb +19 -8
  139. data/lib/html2rss/selectors/README.md +1 -1
  140. data/lib/html2rss/selectors/option_spec.rb +4 -1
  141. data/lib/html2rss/selectors/post_processors/base.rb +0 -8
  142. data/lib/html2rss/selectors/post_processors/gsub.rb +7 -7
  143. data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
  144. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
  145. data/lib/html2rss/selectors/post_processors/parse_time.rb +3 -3
  146. data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
  147. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
  148. data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
  149. data/lib/html2rss/selectors.rb +53 -12
  150. data/lib/html2rss/sst/attrs.rb +2 -2
  151. data/lib/html2rss/sst/document.rb +3 -3
  152. data/lib/html2rss/sst/index.rb +30 -11
  153. data/lib/html2rss/sst/node.rb +33 -26
  154. data/lib/html2rss/sst/normalizer.rb +31 -19
  155. data/lib/html2rss/sst/tags.rb +113 -2
  156. data/lib/html2rss/sst/text.rb +69 -29
  157. data/lib/html2rss/sst.rb +2 -0
  158. data/lib/html2rss/status.rb +6 -3
  159. data/lib/html2rss/surface_category.rb +1 -1
  160. data/lib/html2rss/syndication/README.md +12 -6
  161. data/lib/html2rss/syndication/alternate_links.rb +1 -1
  162. data/lib/html2rss/syndication/channel_facts.rb +87 -0
  163. data/lib/html2rss/syndication/discovery.rb +82 -28
  164. data/lib/html2rss/syndication/feed.rb +20 -0
  165. data/lib/html2rss/syndication/item_media.rb +103 -0
  166. data/lib/html2rss/syndication/parser.rb +75 -51
  167. data/lib/html2rss/test/enhance_audit.rb +2 -2
  168. data/lib/html2rss/test/item_sample.rb +43 -0
  169. data/lib/html2rss/test.rb +42 -14
  170. data/lib/html2rss/url.rb +115 -14
  171. data/lib/html2rss/version.rb +1 -1
  172. data/lib/html2rss/word_count.rb +53 -0
  173. data/lib/html2rss.rb +17 -8
  174. data/schema/html2rss-config.schema.json +56 -41
  175. metadata +31 -4
  176. data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
  177. data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: html2rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.33.0
4
+ version: 0.34.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gil Desmarais
@@ -270,9 +270,16 @@ files:
270
270
  - lib/html2rss/article.rb
271
271
  - lib/html2rss/article/deduplicator.rb
272
272
  - lib/html2rss/article/enclosure.rb
273
+ - lib/html2rss/article/title.rb
273
274
  - lib/html2rss/auto_source.rb
274
275
  - lib/html2rss/auto_source/README.md
275
276
  - lib/html2rss/auto_source/cleanup.rb
277
+ - lib/html2rss/auto_source/corroboration.rb
278
+ - lib/html2rss/auto_source/page_links.rb
279
+ - lib/html2rss/auto_source/page_links/anchor_claims.rb
280
+ - lib/html2rss/auto_source/page_links/card_units.rb
281
+ - lib/html2rss/auto_source/page_links/label_binding.rb
282
+ - lib/html2rss/auto_source/page_links/sponsored_cards.rb
276
283
  - lib/html2rss/auto_source/scraper.rb
277
284
  - lib/html2rss/auto_source/scraper/html.rb
278
285
  - lib/html2rss/auto_source/scraper/json_document.rb
@@ -324,10 +331,12 @@ files:
324
331
  - lib/html2rss/config/validation_issue.rb
325
332
  - lib/html2rss/config/validation_report.rb
326
333
  - lib/html2rss/config/validator.rb
334
+ - lib/html2rss/contract_error.rb
327
335
  - lib/html2rss/defaults.rb
328
336
  - lib/html2rss/doctor.rb
329
337
  - lib/html2rss/doctor/botasaurus.rb
330
338
  - lib/html2rss/error.rb
339
+ - lib/html2rss/error_kind.rb
331
340
  - lib/html2rss/feed_builder.rb
332
341
  - lib/html2rss/feed_builder/item_presentation.rb
333
342
  - lib/html2rss/feed_builder/json_feed.rb
@@ -373,14 +382,20 @@ files:
373
382
  - lib/html2rss/html/rendering/description_builder.rb
374
383
  - lib/html2rss/html/rendering/escaped_attributes.rb
375
384
  - lib/html2rss/html/rendering/image_renderer.rb
376
- - lib/html2rss/html/rendering/media_renderer.rb
377
- - lib/html2rss/html/rendering/pdf_renderer.rb
378
385
  - lib/html2rss/html/rendering/video_renderer.rb
379
386
  - lib/html2rss/html/sst_article_extractor.rb
387
+ - lib/html2rss/html/title_pieces.rb
388
+ - lib/html2rss/html/title_pieces/kicker.rb
389
+ - lib/html2rss/html/title_pieces/nodes.rb
390
+ - lib/html2rss/html/title_pieces/nokogiri_nodes.rb
391
+ - lib/html2rss/html/title_pieces/sst_nodes.rb
392
+ - lib/html2rss/html/visibility.rb
380
393
  - lib/html2rss/link_destination.rb
381
394
  - lib/html2rss/link_destination/destination_facts.rb
382
395
  - lib/html2rss/link_destination/noise_policy.rb
383
396
  - lib/html2rss/link_destination/path_classifier.rb
397
+ - lib/html2rss/link_destination/sponsorship.rb
398
+ - lib/html2rss/link_destination/sponsorship/label_text.rb
384
399
  - lib/html2rss/link_destination/text_classifier.rb
385
400
  - lib/html2rss/mcp.rb
386
401
  - lib/html2rss/mcp/README.md
@@ -403,11 +418,17 @@ files:
403
418
  - lib/html2rss/request_service/budget.rb
404
419
  - lib/html2rss/request_service/compressed_body.rb
405
420
  - lib/html2rss/request_service/context.rb
421
+ - lib/html2rss/request_service/cookie_jar.rb
422
+ - lib/html2rss/request_service/cookie_store.rb
423
+ - lib/html2rss/request_service/feed_advertisement.rb
406
424
  - lib/html2rss/request_service/httpx_strategy.rb
407
425
  - lib/html2rss/request_service/local_file_strategy.rb
408
426
  - lib/html2rss/request_service/network_guard.rb
409
427
  - lib/html2rss/request_service/policy.rb
428
+ - lib/html2rss/request_service/replay_breaker.rb
410
429
  - lib/html2rss/request_service/response.rb
430
+ - lib/html2rss/request_service/response_cache.rb
431
+ - lib/html2rss/request_service/response_caching.rb
411
432
  - lib/html2rss/request_service/response_guard.rb
412
433
  - lib/html2rss/request_service/strategy.rb
413
434
  - lib/html2rss/request_session.rb
@@ -442,6 +463,7 @@ files:
442
463
  - lib/html2rss/selectors/post_processors.rb
443
464
  - lib/html2rss/selectors/post_processors/base.rb
444
465
  - lib/html2rss/selectors/post_processors/gsub.rb
466
+ - lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb
445
467
  - lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
446
468
  - lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb
447
469
  - lib/html2rss/selectors/post_processors/markdown_to_html.rb
@@ -466,13 +488,18 @@ files:
466
488
  - lib/html2rss/syndication/README.md
467
489
  - lib/html2rss/syndication/alternate_links.rb
468
490
  - lib/html2rss/syndication/candidate_catalog.rb
491
+ - lib/html2rss/syndication/channel_facts.rb
469
492
  - lib/html2rss/syndication/discovery.rb
493
+ - lib/html2rss/syndication/feed.rb
494
+ - lib/html2rss/syndication/item_media.rb
470
495
  - lib/html2rss/syndication/parser.rb
471
496
  - lib/html2rss/test.rb
472
497
  - lib/html2rss/test/enhance_audit.rb
498
+ - lib/html2rss/test/item_sample.rb
473
499
  - lib/html2rss/test/policy.rb
474
500
  - lib/html2rss/url.rb
475
501
  - lib/html2rss/version.rb
502
+ - lib/html2rss/word_count.rb
476
503
  - lib/tasks/config_schema.rake
477
504
  - schema/html2rss-config.schema.json
478
505
  homepage: https://github.com/html2rss/html2rss
@@ -480,7 +507,7 @@ licenses:
480
507
  - MIT
481
508
  metadata:
482
509
  allowed_push_host: https://rubygems.org
483
- changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.33.0
510
+ changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.34.0
484
511
  rubygems_mfa_required: 'true'
485
512
  rdoc_options: []
486
513
  require_paths:
@@ -1,39 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Html2rss
4
- module Html
5
- module Rendering
6
- # Factory: picks the appropriate renderer for a given enclosure or fallback image.
7
- class MediaRenderer
8
- # @param enclosure [Html2rss::Article::Enclosure, nil]
9
- # @param image [String, Html2rss::Url, nil] Fallback image URL
10
- # @param title [String]
11
- # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
12
- def self.for(enclosure:, image:, title:)
13
- return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
14
- return nil unless enclosure
15
-
16
- create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
17
- end
18
-
19
- # @private
20
- # @param type [String, nil] enclosure MIME type
21
- # @param url [String, Html2rss::Url] enclosure URL
22
- # @param title [String, nil] title used by image renderer
23
- # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
24
- def self.create_renderer_for_type(type, url:, title:)
25
- case type
26
- when %r{^image/}
27
- ImageRenderer.new(url:, title:)
28
- when %r{^video/}
29
- VideoRenderer.new(url:, type:)
30
- when %r{^audio/}
31
- AudioRenderer.new(url:, type:)
32
- when 'application/pdf'
33
- PdfRenderer.new(url:)
34
- end
35
- end
36
- end
37
- end
38
- end
39
- end
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Html2rss
4
- module Html
5
- module Rendering
6
- # Renders an HTML <iframe> for PDF documents.
7
- class PdfRenderer
8
- include EscapedAttributes
9
-
10
- # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
11
- def initialize(url:)
12
- @url = url
13
- end
14
-
15
- # @return [String] HTML iframe snippet for PDF rendering
16
- def to_html
17
- attributes = [
18
- %(src="#{escaped_url}"),
19
- 'width="100%"',
20
- 'height="75vh"',
21
- 'sandbox=""',
22
- 'referrerpolicy="no-referrer"',
23
- 'loading="lazy"'
24
- ]
25
- "<iframe #{attributes.join(' ')}></iframe>"
26
- end
27
- end
28
- end
29
- end
30
- end