html2rss 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/lib/html2rss/article/deduplicator.rb +2 -4
  4. data/lib/html2rss/article.rb +16 -39
  5. data/lib/html2rss/auto_source/README.md +1 -1
  6. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  7. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  8. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  10. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  11. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  13. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  14. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  15. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  16. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  17. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper.rb +4 -14
  19. data/lib/html2rss/auto_source/segment.rb +7 -4
  20. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  21. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  22. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  23. data/lib/html2rss/auto_source.rb +2 -2
  24. data/lib/html2rss/batch.rb +3 -1
  25. data/lib/html2rss/capture/README.md +6 -4
  26. data/lib/html2rss/capture.rb +67 -35
  27. data/lib/html2rss/channel.rb +3 -3
  28. data/lib/html2rss/cli/probe_view.rb +5 -3
  29. data/lib/html2rss/cli/render.rb +6 -5
  30. data/lib/html2rss/cli/validate.rb +28 -27
  31. data/lib/html2rss/cli.rb +1 -1
  32. data/lib/html2rss/config/issue_mapper.rb +108 -0
  33. data/lib/html2rss/config/request_headers.rb +2 -3
  34. data/lib/html2rss/config/schema.rb +16 -1
  35. data/lib/html2rss/config/selectors_validator.rb +92 -70
  36. data/lib/html2rss/config/validation_issue.rb +60 -0
  37. data/lib/html2rss/config/validation_report.rb +55 -0
  38. data/lib/html2rss/config/validator.rb +10 -6
  39. data/lib/html2rss/config.rb +35 -113
  40. data/lib/html2rss/defaults.rb +42 -30
  41. data/lib/html2rss/doctor/botasaurus.rb +6 -4
  42. data/lib/html2rss/doctor.rb +8 -0
  43. data/lib/html2rss/error.rb +1 -1
  44. data/lib/html2rss/feed_builder/rss.rb +9 -14
  45. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  46. data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
  47. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  48. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  49. data/lib/html2rss/feed_resolution/options.rb +3 -1
  50. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  51. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  52. data/lib/html2rss/html/article_extractor.rb +2 -4
  53. data/lib/html2rss/html/feed_link.rb +3 -1
  54. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  55. data/lib/html2rss/html/navigator.rb +10 -10
  56. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  57. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  58. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  59. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  60. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  61. data/lib/html2rss/mcp/README.md +2 -0
  62. data/lib/html2rss/mcp/config_argument.rb +3 -1
  63. data/lib/html2rss/mcp/contract.rb +6 -20
  64. data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
  65. data/lib/html2rss/mcp/outcome.rb +36 -43
  66. data/lib/html2rss/mcp/server/tools.rb +111 -131
  67. data/lib/html2rss/mcp/server.rb +26 -18
  68. data/lib/html2rss/page_recon/README.md +5 -5
  69. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  70. data/lib/html2rss/page_recon.rb +9 -5
  71. data/lib/html2rss/recon.rb +19 -30
  72. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  73. data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
  74. data/lib/html2rss/request_service/compressed_body.rb +4 -1
  75. data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
  76. data/lib/html2rss/request_service/network_guard.rb +7 -5
  77. data/lib/html2rss/request_service/response.rb +1 -2
  78. data/lib/html2rss/request_service/strategy.rb +3 -8
  79. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  80. data/lib/html2rss/scoring/engine.rb +8 -8
  81. data/lib/html2rss/scoring/observation.rb +5 -3
  82. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  83. data/lib/html2rss/scrape_target.rb +3 -1
  84. data/lib/html2rss/selector_candidates.rb +234 -0
  85. data/lib/html2rss/selectors/README.md +64 -0
  86. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  87. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  88. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  89. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  90. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  91. data/lib/html2rss/selectors/extractors.rb +33 -20
  92. data/lib/html2rss/selectors/item_env.rb +31 -0
  93. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  94. data/lib/html2rss/selectors/option_spec.rb +140 -0
  95. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  96. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  97. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  98. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  99. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  100. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  101. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  102. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  103. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  104. data/lib/html2rss/selectors/post_processors.rb +6 -5
  105. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  106. data/lib/html2rss/selectors/step_env.rb +24 -0
  107. data/lib/html2rss/selectors.rb +231 -142
  108. data/lib/html2rss/sst/attrs.rb +5 -3
  109. data/lib/html2rss/sst/document.rb +3 -1
  110. data/lib/html2rss/sst/index.rb +3 -1
  111. data/lib/html2rss/sst/node.rb +29 -10
  112. data/lib/html2rss/sst/normalizer.rb +12 -16
  113. data/lib/html2rss/sst/tags.rb +7 -7
  114. data/lib/html2rss/status.rb +13 -8
  115. data/lib/html2rss/surface_category.rb +14 -32
  116. data/lib/html2rss/syndication/parser.rb +2 -2
  117. data/lib/html2rss/test/enhance_audit.rb +1 -3
  118. data/lib/html2rss/test/policy.rb +106 -0
  119. data/lib/html2rss/test.rb +126 -173
  120. data/lib/html2rss/url.rb +25 -4
  121. data/lib/html2rss/version.rb +1 -1
  122. data/lib/html2rss.rb +21 -13
  123. data/lib/tasks/config_schema.rake +2 -2
  124. data/schema/html2rss-config.schema.json +11 -8
  125. metadata +15 -6
  126. data/lib/html2rss/selectors/item_scope.rb +0 -30
@@ -4,8 +4,8 @@ module Html2rss
4
4
  class AutoSource
5
5
  class Segmenter
6
6
  ##
7
- # Builds repeated-list article segments from tag_path frequency
8
- # (port of Discovery::ListCandidates using tag_path instead of xpath).
7
+ # Builds repeated-list article segments by grouping cards under a shared parent.
8
+ # When the document has a +main+ or +[role=main]+, anchors outside it are dropped.
9
9
  module List
10
10
  module_function
11
11
 
@@ -30,40 +30,55 @@ module Html2rss
30
30
  end
31
31
 
32
32
  def article_pairs(segmenter)
33
- by_path = relevant_links_by_path(segmenter)
34
- top_paths(segmenter, by_path).flat_map do |path|
35
- by_path.fetch(path, []).filter_map do |node|
36
- article_tag = parent_until_boundary(segmenter, node)
37
- next unless article_tag
38
-
39
- [article_tag, node]
40
- end
41
- end
33
+ top_groups(segmenter).flatten(1)
42
34
  end
43
35
  module_function :article_pairs
44
36
  private_class_method :article_pairs
45
37
 
46
- def relevant_links_by_path(segmenter)
47
- by_path = Hash.new { |hash, path| hash[path] = [] }
48
- segmenter.index.each_node do |node|
38
+ def top_groups(segmenter)
39
+ cards_by_parent(segmenter)
40
+ .select { |_parent, cards| cards.size >= segmenter.minimum_selector_frequency }
41
+ .max_by(segmenter.use_top_selectors) { |_parent, cards| cards.size }
42
+ .map(&:last)
43
+ end
44
+ module_function :top_groups
45
+ private_class_method :top_groups
46
+
47
+ def cards_by_parent(segmenter) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
48
+ index = segmenter.index
49
+ main = main_landmark(index)
50
+ groups = {}.compare_by_identity
51
+
52
+ index.each_node do |node|
49
53
  next unless node.link?
50
- next if segmenter.index.ignored_chrome?(node)
54
+ next if index.ignored_chrome?(node)
51
55
  next unless relevant_anchor?(segmenter, node)
56
+ next if main && !inside_main?(index, node, main)
57
+
58
+ card = parent_until_boundary(segmenter, node)
59
+ next unless card
52
60
 
53
- by_path[node.tag_path] << node
61
+ parent = index.parent_of(card)
62
+ next unless parent
63
+
64
+ (groups[parent] ||= []) << [card, node]
54
65
  end
55
- by_path
66
+ groups
67
+ end
68
+ module_function :cards_by_parent
69
+ private_class_method :cards_by_parent
70
+
71
+ def main_landmark(index)
72
+ index.each_node.find { |node| node.name == :main || node.attrs.raw['role'] == 'main' }
56
73
  end
57
- module_function :relevant_links_by_path
58
- private_class_method :relevant_links_by_path
74
+ module_function :main_landmark
75
+ private_class_method :main_landmark
59
76
 
60
- def top_paths(segmenter, by_path)
61
- by_path.select { |_path, nodes| nodes.size >= segmenter.minimum_selector_frequency }
62
- .max_by(segmenter.use_top_selectors) { |_path, nodes| nodes.size }
63
- .map(&:first)
77
+ def inside_main?(index, node, main)
78
+ node.equal?(main) || index.descendant_of?(node, main)
64
79
  end
65
- module_function :top_paths
66
- private_class_method :top_paths
80
+ module_function :inside_main?
81
+ private_class_method :inside_main?
67
82
 
68
83
  def relevant_anchor?(segmenter, node)
69
84
  facts = segmenter.link_resolver.destination_facts(node)
@@ -77,12 +92,15 @@ module Html2rss
77
92
  module_function :relevant_anchor?
78
93
  private_class_method :relevant_anchor?
79
94
 
95
+ BOUNDARY_TAGS = Set[:body, :html].freeze
96
+ private_constant :BOUNDARY_TAGS
97
+
80
98
  def parent_until_boundary(segmenter, node)
81
99
  index = segmenter.index
82
- link_counts = Hash.new { |hash, curr| hash[curr] = count_links(curr) }
100
+ link_counts = Hash.new { |hash, curr| hash[curr] = count_relevant_links(segmenter, curr) }
83
101
 
84
102
  index.parent_until(node, lambda { |curr|
85
- return true if %i[body html].include?(curr.name)
103
+ return true if BOUNDARY_TAGS.include?(curr.name)
86
104
  return false if index.ignored_chrome?(curr)
87
105
 
88
106
  parent = index.parent_of(curr)
@@ -92,11 +110,16 @@ module Html2rss
92
110
  module_function :parent_until_boundary
93
111
  private_class_method :parent_until_boundary
94
112
 
95
- def count_links(node)
96
- node.link? ? 1 : node.descendants.count(&:link?)
113
+ def count_relevant_links(segmenter, node)
114
+ # Chrome inside a card must not stop the walk before the shared list parent.
115
+ if node.link?
116
+ relevant_anchor?(segmenter, node) ? 1 : 0
117
+ else
118
+ node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
119
+ end
97
120
  end
98
- module_function :count_links
99
- private_class_method :count_links
121
+ module_function :count_relevant_links
122
+ private_class_method :count_relevant_links
100
123
  end
101
124
  end
102
125
  end
@@ -7,7 +7,7 @@ module Html2rss
7
7
  #
8
8
  # Strategies:
9
9
  # - +:semantic+ — leaf article/section/li/tr/div containers with a primary link
10
- # - +:list+ — repeated tag_path anchors walked to a shared container boundary
10
+ # - +:list+ — repeated cards grouped by parent (`main` is a filter when present)
11
11
  # - +:cluster+ — class / structure clustering for anchorless card grids
12
12
  class Segmenter
13
13
  # @param document [SST::Document]
@@ -68,11 +68,11 @@ module Html2rss
68
68
  # @param container [SST::Node]
69
69
  # @return [Boolean]
70
70
  def landmark_ancestor?(anchor, container)
71
- cache = (@landmark_cache ||= {})
72
- key = [anchor.object_id, container.object_id]
73
- return cache[key] if cache.key?(key)
71
+ cache = (@landmark_cache ||= {}.compare_by_identity)
72
+ anchor_cache = (cache[anchor] ||= {}.compare_by_identity)
73
+ return anchor_cache[container] if anchor_cache.key?(container)
74
74
 
75
- cache[key] = compute_landmark_ancestor?(anchor:, container:)
75
+ anchor_cache[container] = compute_landmark_ancestor?(anchor:, container:)
76
76
  end
77
77
 
78
78
  private
@@ -93,7 +93,7 @@ module Html2rss
93
93
  Scraper::SCRAPERS.sum do |scraper|
94
94
  if config.dig(:scraper, scraper.options_key, :enabled)
95
95
  opts = config.dig(:scraper, scraper.options_key)
96
- scraper.respond_to?(:request_slots) ? scraper.request_slots(opts) : 0
96
+ scraper.request_slots(opts)
97
97
  else
98
98
  0
99
99
  end
@@ -180,7 +180,7 @@ module Html2rss
180
180
  captured_responses:
181
181
  )
182
182
  next unless instance
183
- next unless Scraper.extractable_instance?(instance, parsed_body)
183
+ next unless instance.extractable?
184
184
 
185
185
  matched = true
186
186
  articles.concat(run_scraper(instance))
@@ -10,8 +10,10 @@ module Html2rss
10
10
  # Default worker threads for batch operations.
11
11
  DEFAULT_CONCURRENCY = 5
12
12
 
13
+ BatchResult = Data.define(:total, :successful, :results)
14
+
13
15
  # Value object representing the outcome of a batch operation.
14
- BatchResult = Data.define(:total, :successful, :results) do
16
+ class BatchResult
15
17
  ##
16
18
  # @return [Hash{Symbol => Object}]
17
19
  def to_h
@@ -27,7 +27,9 @@ File.write('my-feed.yml', result.yaml)
27
27
  rss = Html2rss.apply(result.config)
28
28
  ```
29
29
 
30
- `Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`).
30
+ `Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`, `candidates`).
31
+
32
+ `#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
31
33
 
32
34
  **Wire vs internal:** user-facing **`Html2rss.apply`** ships RSS from a config Hash. Pipeline internals **`Html2rss.feed` / `feed_result`** stay unchanged — use `apply` in CLI/MCP/docs examples, not `feed`.
33
35
 
@@ -58,13 +60,13 @@ html2rss capture https://example.com --explain # quality JSON on stderr; YAML
58
60
  1. **Request** — `FeedPipeline` (AutoFallback when `:auto`)
59
61
  2. **Discover** — AutoSource extracts admitted articles
60
62
  3. **Segment** — try SST Segmenter strategies `:list` → `:cluster` → `:semantic`
61
- 4. **Gate** — emit items selector only when ≥ `MIN_SELECTOR_MATCHES` (2) articles match
62
- 5. **Assemble** — `{ items: { selector:, enhance: true } }` plus channel
63
+ 4. **Gate** — keep a derived items selector when ≥ `MIN_SELECTOR_MATCHES` (2) articles match
64
+ 5. **Assemble** — `{ items: { selector:, enhance: } }` plus channel. HTML derivation misses fall back to `Selectors::DEFAULT_ITEMS_SELECTOR` (`segment_strategy: :default`) instead of omitting selectors.
63
65
 
64
66
  ## Constraints
65
67
 
66
68
  - Selector quality depends on page structure; treat output as a first draft.
67
- - When the quality gate fails, selectors are omitted (`has_selectors: false`) rather than inventing attribute selectors.
69
+ - When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs, or a rescued `ArgumentError`), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
68
70
  - Botasaurus hops need `BOTASAURUS_SCRAPER_URL`.
69
71
 
70
72
  See also {Html2rss::AutoSource} for article discovery and {Html2rss::FeedPipeline} for the `:auto` request chain.
@@ -25,12 +25,15 @@ module Html2rss
25
25
  CHROME_DROP_THRESHOLD = 3
26
26
  private_constant :CHROME_DROP_THRESHOLD
27
27
 
28
- ##
29
- # Result of a capture operation (config plus quality meta).
30
28
  CaptureResult = Data.define(
31
29
  :config, :yaml, :articles_count, :channel_title, :has_selectors, :segment_strategy,
32
- :admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url
33
- ) do
30
+ :admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url,
31
+ :candidates
32
+ )
33
+
34
+ ##
35
+ # Result of a capture operation (config plus quality meta).
36
+ class CaptureResult
34
37
  # rubocop:disable Metrics/ParameterLists
35
38
  ##
36
39
  # @param config [Hash]
@@ -44,9 +47,10 @@ module Html2rss
44
47
  # @param inferred_topics [Array<String>]
45
48
  # @param native_feed [String, nil]
46
49
  # @param suggested_channel_url [String, nil]
50
+ # @param candidates [Hash{Symbol=>Array}, nil] ranked selector buckets from {SelectorCandidates}
47
51
  def initialize(config:, articles_count:, channel_title:, has_selectors:, segment_strategy:, # rubocop:disable Metrics/MethodLength
48
52
  yaml: nil, admission_drops: {}, selected_strategy: nil, inferred_topics: [], native_feed: nil,
49
- suggested_channel_url: nil)
53
+ suggested_channel_url: nil, candidates: nil)
50
54
  super(
51
55
  config:,
52
56
  yaml: yaml || "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
@@ -58,7 +62,8 @@ module Html2rss
58
62
  selected_strategy:,
59
63
  inferred_topics:,
60
64
  native_feed:,
61
- suggested_channel_url:
65
+ suggested_channel_url:,
66
+ candidates: candidates || SelectorCandidates.empty
62
67
  )
63
68
  end
64
69
  # rubocop:enable Metrics/ParameterLists
@@ -127,7 +132,7 @@ module Html2rss
127
132
  def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
128
133
  outcome = FeedPipeline.new(raw_config).to_outcome
129
134
  @admission_drops = outcome.admission_drops
130
- selectors, segment_strategy = derive_selectors(outcome.response, outcome.articles)
135
+ selectors, segment_strategy, candidates = derive_selectors(outcome.response, outcome.articles)
131
136
  ch_title = @title || channel_title_from(outcome.response)
132
137
  topics = @topics || infer_topics("#{@url} #{ch_title}")
133
138
  native_feed = probe_native_feed(outcome.response) unless @force
@@ -151,7 +156,8 @@ module Html2rss
151
156
  selected_strategy: outcome.selected_strategy,
152
157
  inferred_topics: topics,
153
158
  native_feed:,
154
- suggested_channel_url: suggested_channel_url(outcome)
159
+ suggested_channel_url: suggested_channel_url(outcome),
160
+ candidates:
155
161
  )
156
162
  end
157
163
 
@@ -254,40 +260,68 @@ module Html2rss
254
260
  { strategy: :local_file, request: { local_file_path: @local_file_path } }
255
261
  end
256
262
 
257
- # @return [Array(Hash, Symbol, nil)] selectors hash and winning segment strategy
263
+ # @return [Array(Hash, Symbol, nil, Hash)] selectors, segment strategy, candidate buckets
258
264
  def derive_selectors(response, articles)
259
- return hint_selectors if @items_selector_hint
260
- return [{}, nil] if articles.empty? || !response.html_response?
265
+ return [*hint_selectors, SelectorCandidates.empty] if @items_selector_hint
266
+ return [{}, nil, SelectorCandidates.empty] unless response.html_response?
267
+ return [*default_selectors, SelectorCandidates.empty] if articles.empty?
261
268
 
262
269
  sst = SST::Normalizer.call(response.body)
263
- return [{}, nil] unless sst
270
+ return [*default_selectors, SelectorCandidates.empty] unless sst
264
271
 
265
272
  select_enhance_selectors(sst, articles)
266
273
  rescue ArgumentError => error
267
274
  Log.warn("Capture selector derivation failed: #{error.message}")
268
- [{}, nil]
275
+ [*default_selectors, SelectorCandidates.empty]
269
276
  end
270
277
 
271
278
  def hint_selectors
272
279
  [{ items: { selector: @items_selector_hint, enhance: resolve_enhance } }, :hint]
273
280
  end
274
281
 
275
- def select_enhance_selectors(sst, articles) # rubocop:disable Metrics/MethodLength -- strategy loop + gate
282
+ def default_selectors
283
+ [{ items: { selector: Selectors::DEFAULT_ITEMS_SELECTOR, enhance: resolve_enhance } }, :default]
284
+ end
285
+
286
+ def select_enhance_selectors(sst, articles)
287
+ evidence = collect_items_evidence(sst, articles)
288
+ candidates = SelectorCandidates.call(items_evidence: evidence, min_matches: MIN_SELECTOR_MATCHES)
289
+ top = candidates[:items].first
290
+ return [*default_selectors, candidates] unless top
291
+
292
+ strategy = evidence.find { |entry| entry[:selector] == top[:selector] }&.fetch(:strategy)
293
+ items = { selector: top[:selector], enhance: top[:enhance] }
294
+ [{ items: }, strategy, candidates]
295
+ end
296
+
297
+ def collect_items_evidence(sst, articles)
276
298
  link_resolver = Scoring::LinkResolver.new(@url)
277
299
  enhance = resolve_enhance
278
300
 
279
- SEGMENT_STRATEGIES.each do |strategy|
301
+ SEGMENT_STRATEGIES.flat_map do |strategy|
280
302
  segments = AutoSource::Segmenter.call(
281
303
  sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
282
304
  )
283
305
  matched = match_segments_to_articles(segments, articles)
284
- items_sel = items_selector(matched)
285
- next unless items_sel && matched.size >= MIN_SELECTOR_MATCHES
306
+ next [] if matched.size < MIN_SELECTOR_MATCHES
307
+
308
+ items_evidence_for(matched, strategy:, enhance:)
309
+ end
310
+ end
286
311
 
287
- return [{ items: { selector: items_sel, enhance: } }, strategy]
312
+ def items_evidence_for(matched, strategy:, enhance:)
313
+ roots = lift_heading_link_roots(matched.map { |pair| pair[:segment].root_node })
314
+ selector_variants_for(roots).map do |kind, selector|
315
+ { selector:, enhance:, strategy:, kind:, match_count: matched.size, roots: }
288
316
  end
317
+ end
289
318
 
290
- [{}, nil]
319
+ def selector_variants_for(roots)
320
+ [
321
+ [:shared_class, shared_class_items_selector(roots)],
322
+ [:unique_tag, unique_tag_items_selector(roots)],
323
+ [:path, path_items_selector(roots)]
324
+ ].filter_map { |kind, selector| [kind, selector] if selector }
291
325
  end
292
326
 
293
327
  def match_segments_to_articles(segments, articles)
@@ -322,21 +356,6 @@ module Html2rss
322
356
  segment.root_node.visible_text.to_s.strip
323
357
  end
324
358
 
325
- def items_selector(matched)
326
- return nil if matched.empty?
327
-
328
- roots = lift_heading_link_roots(matched.map { |m| m[:segment].root_node })
329
- shared = shared_class_items_selector(roots)
330
- return shared if shared
331
-
332
- paths = roots.map(&:tag_path)
333
- common = common_path_prefix(paths)
334
- tag_path = common.empty? ? paths.first.to_s : common
335
- return nil if tag_path.empty?
336
-
337
- css_from_trimmed_tag_path(tag_path)
338
- end
339
-
340
359
  def lift_heading_link_roots(roots)
341
360
  roots.map { |root| lift_heading_link_root(root, roots) }
342
361
  end
@@ -371,7 +390,6 @@ module Html2rss
371
390
 
372
391
  def wrapping_anchor_root?(node)
373
392
  return false unless node.name.to_s == 'a'
374
- return false unless node.respond_to?(:find)
375
393
 
376
394
  tags = Html::Navigator::WRAPPING_ANCHOR_CHILD_TAGS
377
395
  node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
@@ -396,6 +414,20 @@ module Html2rss
396
414
  "#{roots.first.name}.#{shared.min}"
397
415
  end
398
416
 
417
+ def unique_tag_items_selector(roots)
418
+ names = roots.map { |root| root.name.to_s }.uniq
419
+ names.join(', ') if names.size > 1
420
+ end
421
+
422
+ def path_items_selector(roots)
423
+ paths = roots.map(&:tag_path)
424
+ common = common_path_prefix(paths)
425
+ tag_path = common.empty? ? paths.first.to_s : common
426
+ return if tag_path.empty?
427
+
428
+ css_from_trimmed_tag_path(tag_path)
429
+ end
430
+
399
431
  def css_from_trimmed_tag_path(tag_path)
400
432
  segments = tag_path.split('/').reject(&:empty?)
401
433
  trimmed = trim_leading_items_segments(segments)
@@ -2,8 +2,9 @@
2
2
 
3
3
  require 'time'
4
4
 
5
- # rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
6
5
  module Html2rss
6
+ Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author)
7
+
7
8
  ##
8
9
  # Materialized channel metadata for feed builders and {FeedResult}.
9
10
  #
@@ -27,7 +28,7 @@ module Html2rss
27
28
  # @return [Html2rss::Url, nil]
28
29
  # @!attribute [r] author
29
30
  # @return [String, nil]
30
- Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author) do
31
+ class Channel # rubocop:disable Metrics/ClassLength -- Channel value object owns extract + freeze + Marshal
31
32
  class << self
32
33
  ##
33
34
  # Materializes channel attributes from an HTTP response and optional overrides.
@@ -193,5 +194,4 @@ module Html2rss
193
194
  Time.httpdate(value)
194
195
  end
195
196
  end
196
- # rubocop:enable Metrics/ModuleLength
197
197
  end
@@ -2,8 +2,6 @@
2
2
 
3
3
  module Html2rss
4
4
  class CLI
5
- ##
6
- # Display-only probe facts for inspect and recon text cards.
7
5
  ProbeView = Data.define(
8
6
  :requested,
9
7
  :final,
@@ -15,7 +13,11 @@ module Html2rss
15
13
  :alternate_feeds,
16
14
  :notes,
17
15
  :strategy
18
- ) do
16
+ )
17
+
18
+ ##
19
+ # Display-only probe facts for inspect and recon text cards.
20
+ class ProbeView
19
21
  class << self
20
22
  ##
21
23
  # @param data [Hash] inspect wire payload
@@ -93,9 +93,11 @@ module Html2rss
93
93
  end
94
94
  end
95
95
  else
96
- warn "\e[31m✗ Test failed\e[0m (#{source})"
97
- warn " Error: #{result.error_message}" if result.error_message
98
- result.validation_errors&.each { |k, v| warn " Schema error [#{k}]: #{Array(v).join(', ')}" }
96
+ $stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
97
+ $stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
98
+ result.validation_issues&.each do |issue|
99
+ $stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
100
+ end
99
101
  end
100
102
  end
101
103
 
@@ -115,8 +117,7 @@ module Html2rss
115
117
  if view.final && view.final != view.requested
116
118
  puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
117
119
  end
118
- surface = view.surface.respond_to?(:to_s) ? view.surface.to_s : view.surface
119
- puts " Surface: #{surface} (#{view.articles_count} articles)"
120
+ puts " Surface: #{view.surface} (#{view.articles_count} articles)"
120
121
  end
121
122
  private_class_method :probe_lines
122
123
  end
@@ -29,11 +29,10 @@ module Html2rss
29
29
  if result.success?
30
30
  puts(target_files.size == 1 ? 'Configuration is valid' : "ok #{file}") unless quiet
31
31
  else
32
- error_details = result.errors.to_h
33
- raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
32
+ raise Thor::Error, format_failure(result) if target_files.size == 1
34
33
 
35
34
  warn "FAIL #{file}"
36
- error_details.each { |key, msg| warn " #{key}: #{Array(msg).join(', ')}" }
35
+ result.issues.each { |issue| warn " #{issue}" }
37
36
  failed << file
38
37
  end
39
38
  end
@@ -57,35 +56,37 @@ module Html2rss
57
56
  resolved.uniq
58
57
  end
59
58
 
60
- def run_named_feed(file, feed_name, params:, quiet:)
61
- result = Html2rss.validate(file, feed_name, params:)
62
- raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
59
+ class << self
60
+ private
63
61
 
64
- puts 'Configuration is valid' unless quiet
65
- end
66
- module_function :run_named_feed
67
- private_class_method :run_named_feed
68
-
69
- def validate_file(file, params:)
70
- if file == '-'
71
- Html2rss.validate($stdin.read, params:)
72
- else
73
- Html2rss.validate(file, params:)
62
+ def run_named_feed(file, feed_name, params:, quiet:)
63
+ result = Html2rss.validate(file, feed_name, params:)
64
+ raise Thor::Error, format_failure(result) unless result.success?
65
+
66
+ puts 'Configuration is valid' unless quiet
67
+ end
68
+
69
+ def validate_file(file, params:)
70
+ if file == '-'
71
+ Html2rss.validate($stdin.read, params:)
72
+ else
73
+ Html2rss.validate(file, params:)
74
+ end
74
75
  end
75
- end
76
- module_function :validate_file
77
- private_class_method :validate_file
78
76
 
79
- def path_like_config?(arg)
80
- path = arg.to_s
81
- return true if path.start_with?('-')
82
- return true if path.end_with?('.yml', '.yaml')
83
- return true if path.include?('/') || path.include?('*')
77
+ def format_failure(report)
78
+ "Invalid configuration: #{report}"
79
+ end
84
80
 
85
- false
81
+ def path_like_config?(arg)
82
+ path = arg.to_s
83
+ return true if path.start_with?('-')
84
+ return true if path.end_with?('.yml', '.yaml')
85
+ return true if path.include?('/') || path.include?('*')
86
+
87
+ false
88
+ end
86
89
  end
87
- module_function :path_like_config?
88
- private_class_method :path_like_config?
89
90
  end
90
91
  end
91
92
  end
data/lib/html2rss/cli.rb CHANGED
@@ -300,7 +300,7 @@ module Html2rss
300
300
  def filter_recon_results(results, verdict_filter)
301
301
  return results unless verdict_filter
302
302
 
303
- expected = Recon::Verdict.coerce(verdict_filter.downcase)
303
+ expected = Recon::Verdict[verdict_filter.downcase]
304
304
  results.select { |r| r.verdict == expected }
305
305
  rescue ArgumentError => error
306
306
  raise Thor::Error, error.message
@@ -0,0 +1,108 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ class Config
5
+ ##
6
+ # Maps internal Dry::Validation messages to {ValidationIssue} (predicate → code).
7
+ # Not a public API — Dry stays inside Validator / this mapper.
8
+ module IssueMapper
9
+ # Dry schema predicates → closed {ValidationIssue::CODES}.
10
+ PREDICATE_CODES = {
11
+ key?: :missing_key,
12
+ filled?: :missing_key,
13
+ str?: :type_mismatch,
14
+ int?: :type_mismatch,
15
+ bool?: :type_mismatch,
16
+ hash?: :type_mismatch,
17
+ array?: :type_mismatch,
18
+ type?: :type_mismatch,
19
+ format?: :invalid_value,
20
+ included_in?: :unknown_value,
21
+ gt?: :constraint,
22
+ gteq?: :constraint,
23
+ lt?: :constraint,
24
+ lteq?: :constraint,
25
+ max_size?: :constraint,
26
+ min_size?: :constraint
27
+ }.freeze
28
+
29
+ class << self
30
+ ##
31
+ # @param dry_result [Dry::Validation::Result]
32
+ # @param values [Hash, nil] config under validation (for actual/expected dig)
33
+ # @return [Html2rss::Config::ValidationReport]
34
+ def from(dry_result, values: nil)
35
+ return ValidationReport.ok if dry_result.success?
36
+
37
+ ValidationReport.failure(dry_result.errors.map { |message| issue_for(message, values:) })
38
+ end
39
+
40
+ ##
41
+ # @param message [String]
42
+ # @return [Html2rss::Config::ValidationReport]
43
+ def parse_failure(message)
44
+ ValidationReport.failure(
45
+ [
46
+ ValidationIssue.new(path: %i[parse], code: :parse, message:)
47
+ ]
48
+ )
49
+ end
50
+
51
+ private
52
+
53
+ def issue_for(message, values:)
54
+ path = Array(message.path).compact
55
+ ValidationIssue.new(
56
+ path:,
57
+ code: code_for(message.predicate),
58
+ message: message.text,
59
+ expected: expected_for(path, values),
60
+ actual: actual_for(path, values, message)
61
+ )
62
+ end
63
+
64
+ def code_for(predicate)
65
+ return :invalid_value if predicate.nil?
66
+
67
+ PREDICATE_CODES.fetch(predicate) do
68
+ raise ArgumentError, "unmapped Dry validation predicate: #{predicate.inspect}"
69
+ end
70
+ end
71
+
72
+ def actual_for(path, values, message)
73
+ dug = dig_path(values, path)
74
+ return dug unless dug.nil?
75
+ return message.input if message.is_a?(Dry::Schema::Message) && !message.input.nil?
76
+
77
+ nil
78
+ end
79
+
80
+ def expected_for(path, values)
81
+ return if values.nil? || path.empty?
82
+ return unless path.last.is_a?(Symbol)
83
+
84
+ parent = dig_path(values, path[0...-1])
85
+ return unless parent.is_a?(Hash)
86
+
87
+ option = Selectors::OptionSpec.option_for(parent, path.last)
88
+ return unless option
89
+
90
+ {
91
+ type: option.json_type,
92
+ required: option.required
93
+ }
94
+ end
95
+
96
+ def dig_path(values, path)
97
+ return if values.nil? || path.empty?
98
+
99
+ path.reduce(values) do |cursor, segment|
100
+ break unless cursor.is_a?(Hash) || (cursor.is_a?(Array) && segment.is_a?(Integer))
101
+
102
+ cursor[segment]
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
108
+ end