html2rss 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/lib/html2rss/article/deduplicator.rb +2 -4
  4. data/lib/html2rss/article.rb +16 -39
  5. data/lib/html2rss/auto_source/README.md +1 -1
  6. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  7. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  8. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  10. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  11. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  13. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  14. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  15. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  16. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  17. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper.rb +4 -14
  19. data/lib/html2rss/auto_source/segment.rb +7 -4
  20. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  21. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  22. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  23. data/lib/html2rss/auto_source.rb +2 -2
  24. data/lib/html2rss/batch.rb +3 -1
  25. data/lib/html2rss/capture/README.md +6 -4
  26. data/lib/html2rss/capture.rb +67 -35
  27. data/lib/html2rss/channel.rb +3 -3
  28. data/lib/html2rss/cli/probe_view.rb +5 -3
  29. data/lib/html2rss/cli/render.rb +6 -5
  30. data/lib/html2rss/cli/validate.rb +28 -27
  31. data/lib/html2rss/cli.rb +1 -1
  32. data/lib/html2rss/config/issue_mapper.rb +108 -0
  33. data/lib/html2rss/config/request_headers.rb +2 -3
  34. data/lib/html2rss/config/schema.rb +16 -1
  35. data/lib/html2rss/config/selectors_validator.rb +92 -70
  36. data/lib/html2rss/config/validation_issue.rb +60 -0
  37. data/lib/html2rss/config/validation_report.rb +55 -0
  38. data/lib/html2rss/config/validator.rb +10 -6
  39. data/lib/html2rss/config.rb +35 -113
  40. data/lib/html2rss/defaults.rb +42 -30
  41. data/lib/html2rss/doctor/botasaurus.rb +6 -4
  42. data/lib/html2rss/doctor.rb +8 -0
  43. data/lib/html2rss/error.rb +1 -1
  44. data/lib/html2rss/feed_builder/rss.rb +9 -14
  45. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  46. data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
  47. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  48. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  49. data/lib/html2rss/feed_resolution/options.rb +3 -1
  50. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  51. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  52. data/lib/html2rss/html/article_extractor.rb +2 -4
  53. data/lib/html2rss/html/feed_link.rb +3 -1
  54. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  55. data/lib/html2rss/html/navigator.rb +10 -10
  56. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  57. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  58. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  59. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  60. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  61. data/lib/html2rss/mcp/README.md +2 -0
  62. data/lib/html2rss/mcp/config_argument.rb +3 -1
  63. data/lib/html2rss/mcp/contract.rb +6 -20
  64. data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
  65. data/lib/html2rss/mcp/outcome.rb +36 -43
  66. data/lib/html2rss/mcp/server/tools.rb +111 -131
  67. data/lib/html2rss/mcp/server.rb +26 -18
  68. data/lib/html2rss/page_recon/README.md +5 -5
  69. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  70. data/lib/html2rss/page_recon.rb +9 -5
  71. data/lib/html2rss/recon.rb +19 -30
  72. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  73. data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
  74. data/lib/html2rss/request_service/compressed_body.rb +4 -1
  75. data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
  76. data/lib/html2rss/request_service/network_guard.rb +7 -5
  77. data/lib/html2rss/request_service/response.rb +1 -2
  78. data/lib/html2rss/request_service/strategy.rb +3 -8
  79. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  80. data/lib/html2rss/scoring/engine.rb +8 -8
  81. data/lib/html2rss/scoring/observation.rb +5 -3
  82. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  83. data/lib/html2rss/scrape_target.rb +3 -1
  84. data/lib/html2rss/selector_candidates.rb +234 -0
  85. data/lib/html2rss/selectors/README.md +64 -0
  86. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  87. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  88. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  89. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  90. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  91. data/lib/html2rss/selectors/extractors.rb +33 -20
  92. data/lib/html2rss/selectors/item_env.rb +31 -0
  93. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  94. data/lib/html2rss/selectors/option_spec.rb +140 -0
  95. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  96. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  97. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  98. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  99. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  100. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  101. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  102. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  103. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  104. data/lib/html2rss/selectors/post_processors.rb +6 -5
  105. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  106. data/lib/html2rss/selectors/step_env.rb +24 -0
  107. data/lib/html2rss/selectors.rb +231 -142
  108. data/lib/html2rss/sst/attrs.rb +5 -3
  109. data/lib/html2rss/sst/document.rb +3 -1
  110. data/lib/html2rss/sst/index.rb +3 -1
  111. data/lib/html2rss/sst/node.rb +29 -10
  112. data/lib/html2rss/sst/normalizer.rb +12 -16
  113. data/lib/html2rss/sst/tags.rb +7 -7
  114. data/lib/html2rss/status.rb +13 -8
  115. data/lib/html2rss/surface_category.rb +14 -32
  116. data/lib/html2rss/syndication/parser.rb +2 -2
  117. data/lib/html2rss/test/enhance_audit.rb +1 -3
  118. data/lib/html2rss/test/policy.rb +106 -0
  119. data/lib/html2rss/test.rb +126 -173
  120. data/lib/html2rss/url.rb +25 -4
  121. data/lib/html2rss/version.rb +1 -1
  122. data/lib/html2rss.rb +21 -13
  123. data/lib/tasks/config_schema.rake +2 -2
  124. data/schema/html2rss-config.schema.json +11 -8
  125. metadata +15 -6
  126. data/lib/html2rss/selectors/item_scope.rb +0 -30
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: html2rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.30.0
4
+ version: 0.31.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gil Desmarais
@@ -314,13 +314,17 @@ files:
314
314
  - lib/html2rss/config.rb
315
315
  - lib/html2rss/config/auto_source_contract.rb
316
316
  - lib/html2rss/config/dynamic_params.rb
317
+ - lib/html2rss/config/issue_mapper.rb
317
318
  - lib/html2rss/config/multiple_feeds_config.rb
318
319
  - lib/html2rss/config/request_controls.rb
319
320
  - lib/html2rss/config/request_headers.rb
320
321
  - lib/html2rss/config/schema.rb
321
322
  - lib/html2rss/config/selectors_validator.rb
323
+ - lib/html2rss/config/validation_issue.rb
324
+ - lib/html2rss/config/validation_report.rb
322
325
  - lib/html2rss/config/validator.rb
323
326
  - lib/html2rss/defaults.rb
327
+ - lib/html2rss/doctor.rb
324
328
  - lib/html2rss/doctor/botasaurus.rb
325
329
  - lib/html2rss/error.rb
326
330
  - lib/html2rss/feed_builder.rb
@@ -421,15 +425,18 @@ files:
421
425
  - lib/html2rss/scoring/ranked_segment.rb
422
426
  - lib/html2rss/scoring/score.rb
423
427
  - lib/html2rss/scrape_target.rb
428
+ - lib/html2rss/selector_candidates.rb
424
429
  - lib/html2rss/selectors.rb
430
+ - lib/html2rss/selectors/README.md
425
431
  - lib/html2rss/selectors/extractors.rb
426
432
  - lib/html2rss/selectors/extractors/attribute.rb
427
433
  - lib/html2rss/selectors/extractors/href.rb
428
434
  - lib/html2rss/selectors/extractors/html.rb
429
435
  - lib/html2rss/selectors/extractors/static.rb
430
436
  - lib/html2rss/selectors/extractors/text.rb
431
- - lib/html2rss/selectors/item_scope.rb
432
- - lib/html2rss/selectors/object_to_xml_converter.rb
437
+ - lib/html2rss/selectors/item_env.rb
438
+ - lib/html2rss/selectors/json_xml.rb
439
+ - lib/html2rss/selectors/option_spec.rb
433
440
  - lib/html2rss/selectors/post_processors.rb
434
441
  - lib/html2rss/selectors/post_processors/base.rb
435
442
  - lib/html2rss/selectors/post_processors/gsub.rb
@@ -441,7 +448,8 @@ files:
441
448
  - lib/html2rss/selectors/post_processors/sanitize_html.rb
442
449
  - lib/html2rss/selectors/post_processors/substring.rb
443
450
  - lib/html2rss/selectors/post_processors/template.rb
444
- - lib/html2rss/selectors/schema_doc.rb
451
+ - lib/html2rss/selectors/schema_export.rb
452
+ - lib/html2rss/selectors/step_env.rb
445
453
  - lib/html2rss/sst.rb
446
454
  - lib/html2rss/sst/attrs.rb
447
455
  - lib/html2rss/sst/document.rb
@@ -459,6 +467,7 @@ files:
459
467
  - lib/html2rss/syndication/parser.rb
460
468
  - lib/html2rss/test.rb
461
469
  - lib/html2rss/test/enhance_audit.rb
470
+ - lib/html2rss/test/policy.rb
462
471
  - lib/html2rss/url.rb
463
472
  - lib/html2rss/version.rb
464
473
  - lib/tasks/config_schema.rake
@@ -468,7 +477,7 @@ licenses:
468
477
  - MIT
469
478
  metadata:
470
479
  allowed_push_host: https://rubygems.org
471
- changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.30.0
480
+ changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.31.0
472
481
  rubygems_mfa_required: 'true'
473
482
  rdoc_options: []
474
483
  require_paths:
@@ -484,7 +493,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
484
493
  - !ruby/object:Gem::Version
485
494
  version: '0'
486
495
  requirements: []
487
- rubygems_version: 4.0.16
496
+ rubygems_version: 4.0.20
488
497
  specification_version: 4
489
498
  summary: Generates RSS feeds from websites by scraping a URL and using CSS selectors
490
499
  to extract item.
data/lib/html2rss/selectors/item_scope.rb DELETED
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Html2rss
4
- class Selectors
5
- ##
6
- # Per-item extraction scope: owns the article node and page +base_url+ for one
7
- # extraction pass. Channel hashes are injected (and cached) by {Selectors}.
8
- #
9
- # Distinct from {Context}, which is the post-processor invocation bag (+options+).
10
- # Post-processor +config+ is derived from {#channel} — not a parallel bag.
11
- ItemScope = Data.define(:item, :base_url, :scraper, :channel) do
12
- ##
13
- # Selects an attribute using this scope's item and base_url.
14
- #
15
- # @param name [Symbol, String]
16
- # @return [Object, Array<Object>]
17
- def select(name) = scraper.select_in_scope(name, self)
18
-
19
- ##
20
- # Builds a post-processor {Context} carrying this scope for nested selects.
21
- #
22
- # @param options [Hash] post-processor options from the selector config
23
- # @option options [String] :name post-processor name
24
- # @return [Context]
25
- def context_for(options:)
26
- Context.new(options:, item:, config: { channel: }, scraper:, item_scope: self)
27
- end
28
- end
29
- end
30
- end