html2rss 0.29.1 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -8
  3. data/html2rss.gemspec +1 -3
  4. data/lib/html2rss/article/deduplicator.rb +2 -4
  5. data/lib/html2rss/article.rb +16 -39
  6. data/lib/html2rss/auto_source/README.md +1 -1
  7. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  8. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  10. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  11. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  13. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  14. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  15. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  16. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  17. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  19. data/lib/html2rss/auto_source/scraper.rb +4 -14
  20. data/lib/html2rss/auto_source/segment.rb +7 -4
  21. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  22. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  23. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  24. data/lib/html2rss/auto_source.rb +2 -2
  25. data/lib/html2rss/batch.rb +6 -4
  26. data/lib/html2rss/capture/README.md +7 -5
  27. data/lib/html2rss/capture.rb +68 -36
  28. data/lib/html2rss/channel.rb +3 -3
  29. data/lib/html2rss/cli/probe_view.rb +5 -3
  30. data/lib/html2rss/cli/render.rb +6 -5
  31. data/lib/html2rss/cli/validate.rb +28 -27
  32. data/lib/html2rss/cli.rb +1 -1
  33. data/lib/html2rss/config/issue_mapper.rb +108 -0
  34. data/lib/html2rss/config/request_headers.rb +6 -4
  35. data/lib/html2rss/config/schema.rb +16 -1
  36. data/lib/html2rss/config/selectors_validator.rb +92 -70
  37. data/lib/html2rss/config/validation_issue.rb +60 -0
  38. data/lib/html2rss/config/validation_report.rb +55 -0
  39. data/lib/html2rss/config/validator.rb +10 -6
  40. data/lib/html2rss/config.rb +35 -101
  41. data/lib/html2rss/defaults.rb +42 -30
  42. data/lib/html2rss/doctor/botasaurus.rb +19 -9
  43. data/lib/html2rss/doctor.rb +8 -0
  44. data/lib/html2rss/error.rb +1 -1
  45. data/lib/html2rss/feed_builder/rss.rb +9 -14
  46. data/lib/html2rss/feed_pipeline/README.md +8 -8
  47. data/lib/html2rss/feed_pipeline/auto_fallback.rb +1 -1
  48. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  49. data/lib/html2rss/feed_pipeline/strategy_plan.rb +7 -3
  50. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  51. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  52. data/lib/html2rss/feed_resolution/options.rb +3 -1
  53. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  54. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  55. data/lib/html2rss/html/article_extractor.rb +2 -4
  56. data/lib/html2rss/html/feed_link.rb +3 -1
  57. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  58. data/lib/html2rss/html/navigator.rb +10 -10
  59. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  60. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  61. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  62. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  63. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  64. data/lib/html2rss/mcp/README.md +5 -3
  65. data/lib/html2rss/mcp/config_argument.rb +3 -1
  66. data/lib/html2rss/mcp/contract.rb +15 -25
  67. data/lib/html2rss/mcp/outcome/playbook.rb +7 -7
  68. data/lib/html2rss/mcp/outcome.rb +36 -43
  69. data/lib/html2rss/mcp/server/tools.rb +112 -132
  70. data/lib/html2rss/mcp/server.rb +27 -19
  71. data/lib/html2rss/page_recon/README.md +5 -5
  72. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  73. data/lib/html2rss/page_recon.rb +9 -5
  74. data/lib/html2rss/recon.rb +20 -31
  75. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  76. data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
  77. data/lib/html2rss/request_service/compressed_body.rb +17 -9
  78. data/lib/html2rss/request_service/httpx_strategy.rb +239 -0
  79. data/lib/html2rss/request_service/network_guard.rb +7 -5
  80. data/lib/html2rss/request_service/response.rb +1 -2
  81. data/lib/html2rss/request_service/response_guard.rb +0 -16
  82. data/lib/html2rss/request_service/strategy.rb +22 -11
  83. data/lib/html2rss/request_service.rb +46 -85
  84. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  85. data/lib/html2rss/scoring/engine.rb +8 -8
  86. data/lib/html2rss/scoring/observation.rb +5 -3
  87. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  88. data/lib/html2rss/scrape_target.rb +3 -1
  89. data/lib/html2rss/selector_candidates.rb +234 -0
  90. data/lib/html2rss/selectors/README.md +64 -0
  91. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  92. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  93. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  94. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  95. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  96. data/lib/html2rss/selectors/extractors.rb +33 -20
  97. data/lib/html2rss/selectors/item_env.rb +31 -0
  98. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  99. data/lib/html2rss/selectors/option_spec.rb +140 -0
  100. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  101. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  102. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  103. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  104. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  105. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  106. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  107. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  108. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  109. data/lib/html2rss/selectors/post_processors.rb +6 -5
  110. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  111. data/lib/html2rss/selectors/step_env.rb +24 -0
  112. data/lib/html2rss/selectors.rb +231 -142
  113. data/lib/html2rss/sst/attrs.rb +5 -3
  114. data/lib/html2rss/sst/document.rb +3 -1
  115. data/lib/html2rss/sst/index.rb +3 -1
  116. data/lib/html2rss/sst/node.rb +29 -10
  117. data/lib/html2rss/sst/normalizer.rb +12 -16
  118. data/lib/html2rss/sst/tags.rb +7 -7
  119. data/lib/html2rss/status.rb +13 -8
  120. data/lib/html2rss/surface_category.rb +14 -32
  121. data/lib/html2rss/syndication/parser.rb +2 -2
  122. data/lib/html2rss/test/enhance_audit.rb +1 -3
  123. data/lib/html2rss/test/policy.rb +106 -0
  124. data/lib/html2rss/test.rb +126 -172
  125. data/lib/html2rss/url.rb +25 -4
  126. data/lib/html2rss/version.rb +1 -1
  127. data/lib/html2rss.rb +27 -19
  128. data/lib/tasks/config_schema.rake +2 -2
  129. data/schema/html2rss-config.schema.json +11 -8
  130. metadata +19 -44
  131. data/lib/html2rss/request_service/faraday_strategy.rb +0 -233
  132. data/lib/html2rss/selectors/item_scope.rb +0 -30
@@ -2,10 +2,12 @@
2
2
 
3
3
  module Html2rss
4
4
  module Html
5
+ FeedLink = Data.define(:href, :mime_type)
6
+
5
7
  # A native RSS/Atom hint from a document head +link[rel=alternate]+.
6
8
  #
7
9
  # Does not guess +/feed+ or +/rss.xml+ paths — that stays in configs +probe_rss+.
8
- FeedLink = Data.define(:href, :mime_type) do
10
+ class FeedLink
9
11
  class << self
10
12
  ##
11
13
  # Collects RSS/Atom +rel=alternate+ links from +doc+'s +head+.
@@ -19,7 +19,7 @@ module Html2rss
19
19
  # @param exclude_nodes [Array<Nokogiri::XML::Node>, nil] nodes to exclude from extraction
20
20
  # @return [String, nil] the concatenated visible text, or nil if none is found
21
21
  def call(tag, separator: ' ', exclude_nodes: nil)
22
- return tag.text.gsub(/\s+/, ' ').strip if tag.respond_to?(:text?) && tag.text?
22
+ return tag.text.gsub(/\s+/, ' ').strip if tag.is_a?(Nokogiri::XML::Text)
23
23
 
24
24
  parts = iterate_children(tag, separator, exclude_nodes)
25
25
  return if parts.empty?
@@ -6,16 +6,16 @@ module Html2rss
6
6
  # Navigator owns DOM chrome recognition and node traversal helpers.
7
7
  class Navigator
8
8
  # Heading tags used to prioritize title extraction and container assessment.
9
- HEADING_TAGS = %w[h1 h2 h3 h4 h5 h6].freeze
9
+ HEADING_TAGS = Set['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].freeze
10
10
 
11
11
  # Element tags that indicate ignored DOM chrome when found in a container path.
12
- IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
12
+ IGNORED_CONTAINER_TAGS = Set['nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
13
13
 
14
14
  # Ancestor tags that usually indicate navigation/utility regions inside a content container.
15
- UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].to_set.freeze
15
+ UTILITY_LANDMARK_TAGS = Set['nav', 'aside', 'footer', 'menu'].freeze
16
16
 
17
17
  # Immediate parent walk stops here — not a usable article card.
18
- CARD_WALK_STOP_TAGS = (UTILITY_LANDMARK_TAGS | IGNORED_CONTAINER_TAGS | %w[html body]).freeze
18
+ CARD_WALK_STOP_TAGS = (UTILITY_LANDMARK_TAGS | IGNORED_CONTAINER_TAGS | Set['html', 'body']).freeze
19
19
 
20
20
  # Inner tags that mean a wrapping <a> is a card, not a span-styled list link.
21
21
  WRAPPING_ANCHOR_CHILD_TAGS = (HEADING_TAGS + %w[p]).freeze
@@ -58,7 +58,7 @@ module Html2rss
58
58
  # @param condition [Proc] The condition to be met.
59
59
  # @return [Nokogiri::XML::Node, nil] The first parent that satisfies the condition.
60
60
  def parent_until_condition(node, condition)
61
- while node && !node.document? && Probe.tag(node) != 'html'
61
+ while node && !node.is_a?(Nokogiri::XML::Document) && Probe.tag(node) != 'html'
62
62
  return node if condition.call(node)
63
63
 
64
64
  node = node.parent
@@ -72,7 +72,7 @@ module Html2rss
72
72
  # @return [Boolean]
73
73
  def usable_card_parent?(node)
74
74
  return false unless node
75
- return false if node.respond_to?(:document?) && node.document?
75
+ return false if node.is_a?(Nokogiri::XML::Document)
76
76
 
77
77
  !CARD_WALK_STOP_TAGS.include?(Probe.tag(node))
78
78
  end
@@ -88,8 +88,7 @@ module Html2rss
88
88
  while current_tag
89
89
  found = current_tag.at_css(selector)
90
90
  return found if found
91
-
92
- return nil unless current_tag.respond_to?(:parent)
91
+ break if current_tag.is_a?(Nokogiri::XML::Document)
93
92
 
94
93
  current_tag = current_tag.parent
95
94
  end
@@ -115,11 +114,12 @@ module Html2rss
115
114
  # @param parent_node [Nokogiri::XML::Node] potential ancestor
116
115
  # @return [Boolean] true when child_node is a descendant of parent_node
117
116
  def descendant_of?(child_node, parent_node)
118
- curr = child_node.respond_to?(:parent) ? child_node.parent : nil
117
+ curr = child_node.is_a?(Nokogiri::XML::Document) ? nil : child_node.parent
119
118
  while curr
120
119
  return true if curr == parent_node
120
+ break if curr.is_a?(Nokogiri::XML::Document)
121
121
 
122
- curr = curr.respond_to?(:parent) ? curr.parent : nil
122
+ curr = curr.parent
123
123
  end
124
124
  false
125
125
  end
@@ -81,7 +81,7 @@ module Html2rss
81
81
 
82
82
  def processed_base_description
83
83
  text = self.class.remove_pattern_from_start(@base, @title)
84
- Html2rss::Selectors::PostProcessors::SanitizeHtml.get(text, @url)
84
+ Html2rss::Selectors::PostProcessors::SanitizeHtml.call(text, @url)
85
85
  end
86
86
  end
87
87
  end
@@ -12,9 +12,9 @@ module Html2rss
12
12
  # CSS class tokens that mark kicker / eyebrow text (excluded from titles).
13
13
  KICKER_CLASS_PATTERN = /kicker|eyebrow|pre-title|pretitle|overline/i
14
14
  # Inline emphasis tags used as title fallbacks when no heading exists.
15
- FALLBACK_HEADING_NAMES = %i[strong b].freeze
15
+ FALLBACK_HEADING_NAMES = Set[:strong, :b].freeze
16
16
  # Nested blocks that mean a category node is actually a content container.
17
- CATEGORY_CONTAINER_NAMES = %i[p article section].to_set.freeze
17
+ CATEGORY_CONTAINER_NAMES = Set[:p, :article, :section].freeze
18
18
 
19
19
  class << self
20
20
  ##
@@ -175,10 +175,13 @@ module Html2rss
175
175
  end
176
176
 
177
177
  def descendant_of?(child, ancestor)
178
- # Walk via root children since Extractor may not hold Index; use tag walk on tree.
179
178
  return true if child.equal?(ancestor)
180
179
 
181
- ancestor.descendants.any? { |d| d.equal?(child) }
180
+ if (index = SST::Index.for_node(child))
181
+ return index.descendant_of?(child, ancestor)
182
+ end
183
+
184
+ !ancestor.find { |d| d.equal?(child) }.nil?
182
185
  end
183
186
 
184
187
  def leftover_lines
@@ -220,7 +223,7 @@ module Html2rss
220
223
  def immediate_card_parent
221
224
  parent = sst_parent
222
225
  index = SST::Index.for_node(@root)
223
- index&.parent_until(parent, method(:usable_walk_parent?))
226
+ index&.parent_until(parent, ->(node) { usable_walk_parent?(node) })
224
227
  end
225
228
 
226
229
  def usable_walk_parent?(node)
@@ -2,7 +2,6 @@
2
2
 
3
3
  module Html2rss
4
4
  module LinkDestination
5
- # Normalized URL plus reusable route-classification facts for one link.
6
5
  DestinationFacts = Data.define(
7
6
  :url,
8
7
  :destination,
@@ -15,7 +14,10 @@ module Html2rss
15
14
  :strong_post_suffix,
16
15
  :high_confidence_junk_path,
17
16
  :high_confidence_utility_destination
18
- ) do
17
+ )
18
+
19
+ # Normalized URL plus reusable route-classification facts for one link.
20
+ class DestinationFacts
19
21
  # @param url [Html2rss::Url] normalized destination URL
20
22
  # @return [DestinationFacts] route facts for downstream link scoring
21
23
  def self.build(url) # rubocop:disable Metrics/MethodLength
@@ -42,7 +42,16 @@ module Html2rss
42
42
  destination_facts.utility_path &&
43
43
  !destination_facts.content_path &&
44
44
  !destination_facts.strong_post_suffix &&
45
- text.to_s.scan(/\p{Alnum}+/).size <= 3
45
+ word_count_at_most?(text, 3)
46
+ end
47
+
48
+ def word_count_at_most?(text, max)
49
+ count = 0
50
+ text.to_s.scan(/\p{Alnum}+/) do
51
+ count += 1
52
+ return false if count > max
53
+ end
54
+ true
46
55
  end
47
56
 
48
57
  def recommended_chrome?(text, destination_facts, heading_anchor:)
@@ -7,47 +7,54 @@ module Html2rss
7
7
  attr_reader :segments
8
8
 
9
9
  # Soft utility segments excluded from high-confidence junk (still utility_path).
10
- SOFT_UTILITY = %w[
11
- for-you join member members membership newsletter newsletters
12
- plans plus premium pricing recommended
13
- ].to_set.freeze
10
+ SOFT_UTILITY = Set[
11
+ 'for-you', 'join', 'member', 'members', 'membership', 'newsletter', 'newsletters',
12
+ 'plans', 'plus', 'premium', 'pricing', 'recommended'
13
+ ].freeze
14
14
 
15
15
  # Segment groups used to classify article, taxonomy, utility, and vanity routes.
16
16
  # Utility = taxonomy ∪ chrome ∪ soft utility (do not re-list taxonomy tokens).
17
17
  SEGMENT_SETS = begin
18
- content = %w[
19
- article articles blog blogs changelog changelogs insight insights
20
- launch launches news post posts release releases story stories update updates
21
- artikel beitrag beitraege nachrichten neuigkeiten aktuelles
22
- articulo articulos noticia noticias entrada entradas publicacion publicaciones
23
- actualite actualites nouvelle nouvelles teaser teasers card cards
24
- ].to_set.freeze
25
- taxonomy = %w[
26
- category categories tag tags topic topics
27
- kategorie kategorien schlagwort schlagworte thema themen
28
- categoria categorias etiqueta etiquetas tema temas
29
- categorie etiquette etiquettes sujet sujets theme themes
30
- ].to_set.freeze
31
- vanity = %w[
32
- join membership plus premium pricing plans subscribe signup
33
- abonnieren abo suscribirse boletin s-abonner saboner
34
- ].to_set.freeze
35
- chrome = %w[
36
- about account archive archives author authors comment comments contact feedback
37
- help login logout notification notifications preference preferences profile register
38
- search settings share signup subscribe feed feeds comment-feed comments-feed privacy
39
- terms cookie cookies user users autor autoren archiv ueber-uns ueber ueberuns profil
40
- kontakt impressum suche hilfe anmelden registrieren konto registrierung anmeldung
41
- abonnieren abo datenschutz nutzungsbedingungen agb autores archivos sobre-nosotros
42
- sobre quienes-somos buscar busqueda ayuda entrar ingresar registrarse registro cuenta
43
- suscribirse boletin privacidad condiciones auteur auteurs a-propos apropos recherche
44
- rechercher aide connexion s-inscrire sinscrire inscription compte s-abonner saboner
45
- lettre-information confidentialite mentions-legales cgu menu sidebar widget social
46
- modal popup banner promo ad ads related recommendation recommendations pagination pager
47
- dating jobs job career careers deals deal shopping shop trading broker versicherung
48
- tierversicherung insurance vergleich comparison partnerboerse singleboerse krypto crypto
49
- casinos casino kreditkarten kreditkarte kredit echtgeld vpn games kaufberater leasing
50
- ].to_set.freeze
18
+ content = Set[
19
+ 'article', 'articles', 'blog', 'blogs', 'changelog', 'changelogs', 'insight', 'insights',
20
+ 'launch', 'launches', 'news', 'post', 'posts', 'release', 'releases', 'story', 'stories',
21
+ 'update', 'updates', 'artikel', 'beitrag', 'beitraege', 'nachrichten', 'neuigkeiten',
22
+ 'aktuelles', 'articulo', 'articulos', 'noticia', 'noticias', 'entrada', 'entradas',
23
+ 'publicacion', 'publicaciones', 'actualite', 'actualites', 'nouvelle', 'nouvelles',
24
+ 'teaser', 'teasers', 'card', 'cards'
25
+ ].freeze
26
+ taxonomy = Set[
27
+ 'category', 'categories', 'tag', 'tags', 'topic', 'topics',
28
+ 'kategorie', 'kategorien', 'schlagwort', 'schlagworte', 'thema', 'themen',
29
+ 'categoria', 'categorias', 'etiqueta', 'etiquetas', 'tema', 'temas',
30
+ 'categorie', 'etiquette', 'etiquettes', 'sujet', 'sujets', 'theme', 'themes'
31
+ ].freeze
32
+ vanity = Set[
33
+ 'join', 'membership', 'plus', 'premium', 'pricing', 'plans', 'subscribe', 'signup',
34
+ 'abonnieren', 'abo', 'suscribirse', 'boletin', 's-abonner', 'saboner'
35
+ ].freeze
36
+ chrome = Set[
37
+ 'about', 'account', 'archive', 'archives', 'author', 'authors', 'comment', 'comments',
38
+ 'contact', 'feedback', 'help', 'login', 'logout', 'notification', 'notifications',
39
+ 'preference', 'preferences', 'profile', 'register', 'search', 'settings', 'share',
40
+ 'signup', 'subscribe', 'feed', 'feeds', 'comment-feed', 'comments-feed', 'privacy',
41
+ 'terms', 'cookie', 'cookies', 'user', 'users', 'autor', 'autoren', 'archiv', 'ueber-uns',
42
+ 'ueber', 'ueberuns', 'profil', 'kontakt', 'impressum', 'suche', 'hilfe', 'anmelden',
43
+ 'registrieren', 'konto', 'registrierung', 'anmeldung', 'abonnieren', 'abo', 'datenschutz',
44
+ 'nutzungsbedingungen', 'agb', 'autores', 'archivos', 'sobre-nosotros', 'sobre',
45
+ 'quienes-somos', 'buscar', 'busqueda', 'ayuda', 'entrar', 'ingresar', 'registrarse',
46
+ 'registro', 'cuenta', 'suscribirse', 'boletin', 'privacidad', 'condiciones', 'auteur',
47
+ 'auteurs', 'a-propos', 'apropos', 'recherche', 'rechercher', 'aide', 'connexion',
48
+ 's-inscrire', 'sinscrire', 'inscription', 'compte', 's-abonner', 'saboner',
49
+ 'lettre-information', 'confidentialite', 'mentions-legales', 'cgu', 'menu', 'sidebar',
50
+ 'widget', 'social', 'modal', 'popup', 'banner', 'promo', 'ad', 'ads', 'related',
51
+ 'recommendation', 'recommendations', 'pagination', 'pager', 'dating', 'jobs', 'job',
52
+ 'career', 'careers', 'deals', 'deal', 'shopping', 'shop', 'trading', 'broker',
53
+ 'versicherung', 'tierversicherung', 'insurance', 'vergleich', 'comparison',
54
+ 'partnerboerse', 'singleboerse', 'krypto', 'crypto', 'casinos', 'casino',
55
+ 'kreditkarten', 'kreditkarte', 'kredit', 'echtgeld', 'vpn', 'games', 'kaufberater',
56
+ 'leasing'
57
+ ].freeze
51
58
  utility = (taxonomy | chrome | SOFT_UTILITY).freeze
52
59
  {
53
60
  content:,
@@ -55,7 +62,7 @@ module Html2rss
55
62
  high_confidence_junk: (utility - SOFT_UTILITY).freeze,
56
63
  taxonomy:,
57
64
  vanity:,
58
- deep_post_context: %w[press newsroom presse pressemitteilungen prensa].to_set.freeze
65
+ deep_post_context: Set['press', 'newsroom', 'presse', 'pressemitteilungen', 'prensa'].freeze
59
66
  }.freeze
60
67
  end
61
68
  # Path segment that begins with a year-like publishing marker.
@@ -20,10 +20,12 @@ Batch: `batch_inspect`, `batch_recon`, `batch_scrape`.
20
20
 
21
21
  ## Decision tree
22
22
 
23
- 1. **Articles now (no saved config)?** → `scrape` (or `batch_scrape` for multiple URLs). `strategy: "auto"` runs Faraday → Botasaurus; do not retry with explicit `faraday` after `auto`.
23
+ 1. **Articles now (no saved config)?** → `scrape` (or `batch_scrape` for multiple URLs). `strategy: "auto"` runs default (HTTPX) → Botasaurus; do not retry with explicit `default` after `auto`.
24
24
  2. **Reusable feed YAML?** → `capture` → `test` → `apply`. `capture` returns YAML in `payload.yaml`. Strive `enhance: true` (false only when chrome leaks). `test` runs schema + live extraction; optional `compare_enhance` compares enhance off vs on. `apply` is the ship gate (`isError` on zero items). Both `test` and `apply` may include `quality_report.enhance_gains` when `selectors.items.enhance` is true.
25
25
  3. **Weak scrape/capture or recon?** → `inspect` (or `batch_inspect`). When alternates warrant it, follow `next_step` to `recon`.
26
26
  4. **Config already in hand?** → `validate` (schema only) → `test` → `apply`.
27
+ - `validate` failures return `payload.issues` (array of `{ path, code, message, expected, actual }`) — not a nested `errors` hash.
28
+ - `test` schema failures surface the same shape under `payload.validation_issues`.
27
29
 
28
30
  Prefer `capture` for durable configs; `scrape` / `batch_scrape` for one-shot extraction. Follow envelope `next_step` and `guidance`; do not parse scrape text as a raw item array.
29
31
 
@@ -61,7 +63,7 @@ Do not duplicate playbook prose in `server.rb`.
61
63
  | ----------------------- | ---------------------------------------------------------- |
62
64
  | `html2rss://schema` | Full JSON Schema for feed configurations |
63
65
  | `html2rss://extractors` | Registered extractor names |
64
- | `html2rss://strategies` | Published MCP strategies (`auto`, `faraday`, `botasaurus`) |
66
+ | `html2rss://strategies` | Published MCP strategies (`auto`, `default`, `httpx`, `botasaurus`) |
65
67
  | `html2rss://runtime` | `version`, `mcp_contract_version`, `catalog_fingerprint`, `tools`, `botasaurus_configured` (never the scraper URL). Fingerprint covers tool names, required keys, and `oneOf` branches; bump `mcp_contract_version` for envelope semantics. |
66
68
 
67
69
  ## Prompts
@@ -73,7 +75,7 @@ Do not duplicate playbook prose in `server.rb`.
73
75
 
74
76
  ## Strategy note
75
77
 
76
- `scrape` / `capture` with `strategy: "auto"` run the full AutoFallback chain. `inspect` maps `auto` to Faraday for cheap diagnostics; pin `botasaurus` when you need browser rendering for inspect.
78
+ `scrape` / `capture` with `strategy: "auto"` run the full AutoFallback chain. `inspect` maps `auto` to default (HTTPX) for cheap diagnostics; pin `botasaurus` when you need browser rendering for inspect.
77
79
 
78
80
  ## Inspect redirects
79
81
 
@@ -2,10 +2,12 @@
2
2
 
3
3
  module Html2rss
4
4
  module MCP
5
+ ConfigArgument = Data.define(:config)
6
+
5
7
  # Feed config supplied over MCP as a hash XOR a YAML string.
6
8
  #
7
9
  # Config does not know MCP. YAML parsing stays on {Html2rss::Config.from_yaml}.
8
- ConfigArgument = Data.define(:config) do
10
+ class ConfigArgument
9
11
  class << self
10
12
  ##
11
13
  # @param config [Hash, nil] feed configuration hash
@@ -9,10 +9,14 @@ module Html2rss
9
9
  # annotations, and the single compact JSON envelope response.
10
10
  module Contract # rubocop:disable Metrics/ModuleLength -- published listing constants stay co-located
11
11
  # Published MCP request strategies (excludes +local_file+).
12
- STRATEGIES = %w[auto faraday botasaurus].freeze
12
+ STRATEGIES = %w[auto default httpx botasaurus].freeze
13
+ # Accepted migration strategies retained for backwards compatibility.
14
+ MIGRATION_STRATEGIES = %w[faraday].freeze
15
+ # Complete set of strategies accepted by {.assert_published_request!}.
16
+ ALL_ACCEPTED_STRATEGIES = (STRATEGIES + MIGRATION_STRATEGIES).freeze
13
17
 
14
18
  # Bump when tool names, required inputs, or envelope semantics change (independent of gem +VERSION+).
15
- MCP_CONTRACT_VERSION = 2
19
+ MCP_CONTRACT_VERSION = 3
16
20
  public_constant :MCP_CONTRACT_VERSION
17
21
 
18
22
  # Raised when apply/validate config uses an unpublished MCP request adapter.
@@ -30,12 +34,12 @@ module Html2rss
30
34
  type: 'string',
31
35
  enum: STRATEGIES,
32
36
  default: 'auto',
33
- description: 'Request strategy (auto runs faraday → botasaurus fallback chain)'
37
+ description: 'Request strategy (auto runs default → botasaurus fallback chain)'
34
38
  }.freeze
35
39
 
36
- # JSON Schema property for inspect +strategy+ (auto stays on Faraday).
40
+ # JSON Schema property for inspect +strategy+ (auto stays on default).
37
41
  INSPECT_STRATEGY_PROPERTY = STRATEGY_PROPERTY.merge(
38
- description: 'Request strategy (auto uses Faraday for cheap diagnostics; pin botasaurus when needed)'
42
+ description: 'Request strategy (auto uses default for cheap diagnostics; pin botasaurus when needed)'
39
43
  ).freeze
40
44
 
41
45
  # JSON Schema +oneOf+ requiring exactly one of +config+ or +yaml+.
@@ -197,26 +201,12 @@ module Html2rss
197
201
  # Tool annotations for +validate+ (closed world).
198
202
  ANNOTATIONS_VALIDATE = ANNOTATIONS_OPEN_WORLD.merge(open_world_hint: false).freeze
199
203
 
200
- # Human titles for +tools/list+.
201
- TITLES = {
202
- scrape: 'Scrape',
203
- inspect: 'Inspect',
204
- recon: 'Recon',
205
- batch_scrape: 'Batch scrape',
206
- batch_inspect: 'Batch inspect',
207
- batch_recon: 'Batch recon',
208
- capture: 'Capture',
209
- validate: 'Validate',
210
- apply: 'Apply',
211
- test: 'Test'
212
- }.freeze
213
-
214
204
  # @api private
215
205
  CATALOG_ENTRY_LINE = lambda do |entry|
216
- schema = entry.fetch(:input_schema)
206
+ schema = entry.input_schema
217
207
  required = Array(schema[:required]).sort.join(',')
218
208
  one_of = Array(schema[:oneOf]).map { |branch| Array(branch[:required]).sort.join('+') }.sort.join('|')
219
- [entry.fetch(:name), required, one_of].reject(&:empty?).join(':')
209
+ [entry.name, required, one_of].reject(&:empty?).join(':')
220
210
  end.freeze
221
211
 
222
212
  class << self
@@ -225,7 +215,7 @@ module Html2rss
225
215
  #
226
216
  # @return [Array<String>]
227
217
  def catalog_tools
228
- Server::Tools::TOOLS.map { |entry| entry.fetch(:name) }.sort
218
+ Server::Tools::TOOLS.map(&:name).sort
229
219
  end
230
220
 
231
221
  ##
@@ -235,7 +225,7 @@ module Html2rss
235
225
  #
236
226
  # @return [String] 16-char hex digest prefix
237
227
  def catalog_fingerprint
238
- lines = Server::Tools::TOOLS.sort_by { |entry| entry.fetch(:name) }.map(&CATALOG_ENTRY_LINE)
228
+ lines = Server::Tools::TOOLS.sort_by(&:name).map(&CATALOG_ENTRY_LINE)
239
229
  Digest::SHA256.hexdigest(lines.join("\n")).slice(0, 16)
240
230
  end
241
231
 
@@ -250,7 +240,7 @@ module Html2rss
250
240
  required: %w[ok next_step guidance payload],
251
241
  properties: {
252
242
  ok: { type: 'boolean' },
253
- next_step: { type: 'string', enum: Outcome::NextStep::NAMES.map(&:to_s) },
243
+ next_step: { type: 'string', enum: Outcome::NextStep::ALL.keys.map(&:to_s) },
254
244
  guidance: { type: 'string' },
255
245
  payload: { type: 'object' }
256
246
  }
@@ -281,7 +271,7 @@ module Html2rss
281
271
  # or +request.local_file_path+ is present
282
272
  def assert_published_request!(config)
283
273
  strategy = config[:strategy]
284
- unless strategy.nil? || STRATEGIES.include?(strategy.to_s)
274
+ unless strategy.nil? || ALL_ACCEPTED_STRATEGIES.include?(strategy.to_s)
285
275
  raise UnpublishedRequestError,
286
276
  "MCP does not accept strategy #{strategy} (published: #{STRATEGIES.join(', ')})"
287
277
  end
@@ -8,7 +8,7 @@ module Html2rss
8
8
  # and prompt bodies. {Server} delegates here — do not duplicate prose in
9
9
  # +server.rb+.
10
10
  class Playbook
11
- # Default guidance copy keyed by {Outcome::NextStep::NAMES}.
11
+ # Default guidance copy; {Outcome::NextStep::ALL} is built from these keys.
12
12
  GUIDANCE = {
13
13
  done: 'Done. Read payload for the result.',
14
14
  inspect: 'Call inspect next. Read payload for diagnostics (final_url, status, ' \
@@ -16,7 +16,7 @@ module Html2rss
16
16
  recon: 'Call recon next. Read payload for verdict and native_feed preference.',
17
17
  validate: 'Call validate with payload.yaml or a config hash (XOR, not both).',
18
18
  apply: 'Call apply next. Confirm payload.item_count before shipping.',
19
- scrape: 'Call scrape for articles now. strategy auto already runs Faraday then Botasaurus ' \
19
+ scrape: 'Call scrape for articles now. strategy auto already runs default (HTTPX) then Botasaurus ' \
20
20
  'and promotes native RSS/Atom when present.',
21
21
  capture: 'Call capture for a reusable YAML draft, then follow next_step.',
22
22
  read_runtime: 'Read html2rss://runtime. Compare mcp_contract_version and catalog_fingerprint ' \
@@ -36,7 +36,7 @@ module Html2rss
36
36
  html2rss MCP — decide which tool to call:
37
37
 
38
38
  1. Need articles now (no saved config)? → scrape (or batch_scrape for multiple)
39
- - strategy "auto" runs Faraday → Botasaurus AutoFallback. Do not retry with explicit faraday after auto.
39
+ - strategy "auto" runs default (HTTPX) → Botasaurus AutoFallback. Do not retry with explicit default after auto.
40
40
  - Empty scrape is still success (articles-now). Follow next_step / guidance (read_runtime if Botasaurus unset).
41
41
  2. Need a reusable feed YAML? → capture → test → apply
42
42
  - capture returns YAML inside payload.yaml. Draft only: if destination is html2rss-configs, rewrite for directory.topics and explicit channel title/url. Default enhance follows capture evidence (false when admission_drops show chrome); override only when needed.
@@ -58,9 +58,9 @@ module Html2rss
58
58
  # @return [String]
59
59
  def scrape_webpage_prompt(url)
60
60
  <<~MSG.strip
61
- Scrape #{url} with scrape (strategy auto). One call is enough — auto already runs Faraday then Botasaurus.
61
+ Scrape #{url} with scrape (strategy auto). One call is enough — auto already runs default then Botasaurus.
62
62
  Follow envelope next_step and guidance. Call inspect only if articles are empty/weak or you need diagnostics (final_url, status, scheme_downgrade, alternate_feeds). When inspect finds alternates, follow next_step to recon.
63
- Do not retry scrape with explicit faraday after auto. Read html2rss://runtime if next_step is read_runtime.
63
+ Do not retry scrape with explicit default after auto. Read html2rss://runtime if next_step is read_runtime.
64
64
  Return payload.items (not a raw JSON array).
65
65
  MSG
66
66
  end
@@ -72,7 +72,7 @@ module Html2rss
72
72
  <<~MSG.strip
73
73
  Build a reusable html2rss feed config for #{url}:
74
74
  1) capture — YAML is payload.yaml. Check payload.articles_count, payload.has_selectors, and payload.suggested_channel_url. enhance defaults from admission evidence (false when chrome drops are high). When payload.native_feed is set, follow next_step (done — use the native feed).
75
- 2) Follow next_step. If weak or you need recon, inspect then recon when alternates warrant it. Auto already hops to Botasaurus; do not retry capture with botasaurus unless Faraday was blocked.
75
+ 2) Follow next_step. If weak or you need recon, inspect then recon when alternates warrant it. Auto already hops to Botasaurus; do not retry capture with botasaurus unless default was blocked.
76
76
  3) test with yaml (or config hash) — schema + live extraction. On :schema failure, validate; on :execution/:min_items, recapture. Read payload.quality_report.enhance_gains when enhance is on; optional compare_enhance compares enhance off vs on without changing shipped RSS.
77
77
  4) apply — isError if zero items. Confirm payload.item_count and payload.quality_report (including enhance_gains) before shipping.
78
78
  If the destination is html2rss-configs, rewrite the draft for directory.topics and explicit channel title/url. Return YAML.
@@ -95,7 +95,7 @@ module Html2rss
95
95
  if data[:blocked_surface] || data[:surface_category].to_s == 'blocked_surface'
96
96
  return 'Blocked or anti-bot interstitial likely. Retry scrape with strategy botasaurus once ' \
97
97
  '(or CLI inspect --deep when BOTASAURUS_SCRAPER_URL is set). ' \
98
- 'Do not retry explicit faraday after auto.'
98
+ 'Do not retry explicit default after auto.'
99
99
  end
100
100
  if data[:likely_js_shell]
101
101
  return 'JS-rendered shell likely (html_present, zero articles). Use strategy auto or botasaurus; ' \