html2rss 0.32.3 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. checksums.yaml +4 -4
  2. data/lib/html2rss/article/enclosure.rb +2 -6
  3. data/lib/html2rss/auto_source/cleanup.rb +1 -3
  4. data/lib/html2rss/auto_source/scraper/html.rb +2 -8
  5. data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
  6. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
  7. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
  8. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  9. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
  10. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
  11. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
  12. data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
  13. data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
  14. data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
  15. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  16. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
  17. data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
  18. data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
  19. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
  20. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  21. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
  22. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  23. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
  25. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
  26. data/lib/html2rss/auto_source/scraper.rb +0 -11
  27. data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
  28. data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
  29. data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
  30. data/lib/html2rss/auto_source.rb +1 -1
  31. data/lib/html2rss/batch.rb +22 -32
  32. data/lib/html2rss/capture/README.md +1 -1
  33. data/lib/html2rss/capture.rb +7 -27
  34. data/lib/html2rss/cli/render.rb +97 -96
  35. data/lib/html2rss/cli/validate.rb +2 -4
  36. data/lib/html2rss/cli.rb +33 -27
  37. data/lib/html2rss/config/request_controls.rb +10 -2
  38. data/lib/html2rss/config/schema.rb +8 -10
  39. data/lib/html2rss/config/selectors_validator.rb +1 -0
  40. data/lib/html2rss/config/validator.rb +3 -3
  41. data/lib/html2rss/config.rb +1 -1
  42. data/lib/html2rss/defaults.rb +5 -16
  43. data/lib/html2rss/doctor/botasaurus.rb +11 -19
  44. data/lib/html2rss/error.rb +54 -1
  45. data/lib/html2rss/feed_builder/rss.rb +1 -1
  46. data/lib/html2rss/feed_pipeline/README.md +1 -1
  47. data/lib/html2rss/feed_pipeline/auto_fallback.rb +58 -55
  48. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  49. data/lib/html2rss/feed_resolution/options.rb +3 -7
  50. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  51. data/lib/html2rss/feed_resolution/probe.rb +3 -3
  52. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  53. data/lib/html2rss/feed_resolution.rb +5 -9
  54. data/lib/html2rss/feed_result.rb +4 -3
  55. data/lib/html2rss/hash_util.rb +91 -90
  56. data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
  57. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  58. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
  59. data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
  60. data/lib/html2rss/html/probe.rb +51 -51
  61. data/lib/html2rss/link_destination/path_classifier.rb +11 -33
  62. data/lib/html2rss/mcp/config_argument.rb +9 -2
  63. data/lib/html2rss/mcp/contract.rb +7 -5
  64. data/lib/html2rss/mcp/outcome.rb +4 -5
  65. data/lib/html2rss/mcp/runtime.rb +17 -24
  66. data/lib/html2rss/mcp/server/tools.rb +9 -9
  67. data/lib/html2rss/option.rb +48 -0
  68. data/lib/html2rss/page_recon/diagnostics.rb +7 -9
  69. data/lib/html2rss/page_recon.rb +0 -4
  70. data/lib/html2rss/recon.rb +10 -12
  71. data/lib/html2rss/request_service/blocked_surface.rb +3 -7
  72. data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
  73. data/lib/html2rss/request_service/budget.rb +3 -24
  74. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  75. data/lib/html2rss/request_service/context.rb +1 -3
  76. data/lib/html2rss/request_service/local_file_strategy.rb +3 -1
  77. data/lib/html2rss/request_service/network_guard.rb +3 -2
  78. data/lib/html2rss/request_service/policy.rb +8 -26
  79. data/lib/html2rss/request_service/response.rb +3 -6
  80. data/lib/html2rss/request_service/strategy.rb +2 -2
  81. data/lib/html2rss/request_service.rb +26 -2
  82. data/lib/html2rss/request_session/pager/base.rb +3 -3
  83. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  84. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  85. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  86. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  87. data/lib/html2rss/scoring/engine.rb +4 -32
  88. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  89. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  90. data/lib/html2rss/selectors/json_xml.rb +33 -34
  91. data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
  92. data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
  93. data/lib/html2rss/selectors/schema_export.rb +60 -60
  94. data/lib/html2rss/selectors.rb +14 -18
  95. data/lib/html2rss/sst/node.rb +9 -3
  96. data/lib/html2rss/sst/normalizer.rb +25 -24
  97. data/lib/html2rss/sst/text.rb +53 -57
  98. data/lib/html2rss/status.rb +23 -30
  99. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  100. data/lib/html2rss/syndication/discovery.rb +117 -187
  101. data/lib/html2rss/syndication/parser.rb +92 -115
  102. data/lib/html2rss/test/enhance_audit.rb +13 -15
  103. data/lib/html2rss/test/policy.rb +9 -11
  104. data/lib/html2rss/test.rb +29 -48
  105. data/lib/html2rss/url.rb +53 -53
  106. data/lib/html2rss/version.rb +1 -1
  107. data/lib/html2rss.rb +2 -1
  108. metadata +4 -2
data/lib/html2rss.rb CHANGED
@@ -101,7 +101,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
101
101
  #
102
102
  # @param config_input [Hash, String] config hash, YAML string, or file path
103
103
  # @param feed_name [String, nil] optional feed name in multi-feed file
104
- # @param min_items [Integer] minimum required items (default: 1)
104
+ # @param min_items [Integer] minimum required items, >= 0 (default: 1)
105
105
  # @param params [Hash] dynamic feed params
106
106
  # @param strategy [Symbol, nil] optional strategy override
107
107
  # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
@@ -109,6 +109,7 @@ module Html2rss # rubocop:disable Metrics/ModuleLength
109
109
  # @return [Html2rss::Test::Result]
110
110
  def self.test(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, strict_quality: false, # rubocop:disable Metrics/ParameterLists
111
111
  compare_enhance: false)
112
+ Option.integer(:min_items, min_items, min: 0)
112
113
  Test.call(config_input, feed_name, min_items:, params:, strategy:, strict_quality:, compare_enhance:)
113
114
  end
114
115
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: html2rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.32.3
4
+ version: 0.33.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gil Desmarais
@@ -391,6 +391,7 @@ files:
391
391
  - lib/html2rss/mcp/runtime.rb
392
392
  - lib/html2rss/mcp/server.rb
393
393
  - lib/html2rss/mcp/server/tools.rb
394
+ - lib/html2rss/option.rb
394
395
  - lib/html2rss/page_recon.rb
395
396
  - lib/html2rss/page_recon/README.md
396
397
  - lib/html2rss/page_recon/diagnostics.rb
@@ -463,6 +464,7 @@ files:
463
464
  - lib/html2rss/surface_category.rb
464
465
  - lib/html2rss/syndication.rb
465
466
  - lib/html2rss/syndication/README.md
467
+ - lib/html2rss/syndication/alternate_links.rb
466
468
  - lib/html2rss/syndication/candidate_catalog.rb
467
469
  - lib/html2rss/syndication/discovery.rb
468
470
  - lib/html2rss/syndication/parser.rb
@@ -478,7 +480,7 @@ licenses:
478
480
  - MIT
479
481
  metadata:
480
482
  allowed_push_host: https://rubygems.org
481
- changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.32.3
483
+ changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.33.0
482
484
  rubygems_mfa_required: 'true'
483
485
  rdoc_options: []
484
486
  require_paths: