html2rss 0.30.0 → 0.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/lib/html2rss/article/deduplicator.rb +2 -4
  4. data/lib/html2rss/article.rb +16 -39
  5. data/lib/html2rss/auto_source/README.md +1 -1
  6. data/lib/html2rss/auto_source/cleanup.rb +15 -6
  7. data/lib/html2rss/auto_source/scraper/html.rb +5 -0
  8. data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
  9. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
  10. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
  11. data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
  12. data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
  13. data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
  14. data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
  15. data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
  16. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
  17. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
  18. data/lib/html2rss/auto_source/scraper.rb +4 -14
  19. data/lib/html2rss/auto_source/segment.rb +7 -4
  20. data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
  21. data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
  22. data/lib/html2rss/auto_source/segmenter.rb +5 -5
  23. data/lib/html2rss/auto_source.rb +2 -2
  24. data/lib/html2rss/batch.rb +3 -1
  25. data/lib/html2rss/capture/README.md +6 -4
  26. data/lib/html2rss/capture.rb +67 -35
  27. data/lib/html2rss/channel.rb +3 -3
  28. data/lib/html2rss/cli/probe_view.rb +5 -3
  29. data/lib/html2rss/cli/render.rb +6 -5
  30. data/lib/html2rss/cli/validate.rb +28 -27
  31. data/lib/html2rss/cli.rb +1 -1
  32. data/lib/html2rss/config/issue_mapper.rb +108 -0
  33. data/lib/html2rss/config/request_headers.rb +2 -3
  34. data/lib/html2rss/config/schema.rb +16 -1
  35. data/lib/html2rss/config/selectors_validator.rb +92 -70
  36. data/lib/html2rss/config/validation_issue.rb +60 -0
  37. data/lib/html2rss/config/validation_report.rb +55 -0
  38. data/lib/html2rss/config/validator.rb +10 -6
  39. data/lib/html2rss/config.rb +35 -113
  40. data/lib/html2rss/defaults.rb +42 -30
  41. data/lib/html2rss/doctor/botasaurus.rb +6 -4
  42. data/lib/html2rss/doctor.rb +8 -0
  43. data/lib/html2rss/error.rb +1 -1
  44. data/lib/html2rss/feed_builder/rss.rb +9 -14
  45. data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
  46. data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
  47. data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
  48. data/lib/html2rss/feed_resolution/diag.rb +3 -1
  49. data/lib/html2rss/feed_resolution/options.rb +3 -1
  50. data/lib/html2rss/feed_resolution/policy.rb +2 -2
  51. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  52. data/lib/html2rss/html/article_extractor.rb +2 -4
  53. data/lib/html2rss/html/feed_link.rb +3 -1
  54. data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
  55. data/lib/html2rss/html/navigator.rb +10 -10
  56. data/lib/html2rss/html/rendering/description_builder.rb +1 -1
  57. data/lib/html2rss/html/sst_article_extractor.rb +8 -5
  58. data/lib/html2rss/link_destination/destination_facts.rb +4 -2
  59. data/lib/html2rss/link_destination/noise_policy.rb +10 -1
  60. data/lib/html2rss/link_destination/path_classifier.rb +45 -38
  61. data/lib/html2rss/mcp/README.md +2 -0
  62. data/lib/html2rss/mcp/config_argument.rb +3 -1
  63. data/lib/html2rss/mcp/contract.rb +6 -20
  64. data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
  65. data/lib/html2rss/mcp/outcome.rb +36 -43
  66. data/lib/html2rss/mcp/server/tools.rb +111 -131
  67. data/lib/html2rss/mcp/server.rb +26 -18
  68. data/lib/html2rss/page_recon/README.md +5 -5
  69. data/lib/html2rss/page_recon/diagnostics.rb +71 -81
  70. data/lib/html2rss/page_recon.rb +9 -5
  71. data/lib/html2rss/recon.rb +19 -30
  72. data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
  73. data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
  74. data/lib/html2rss/request_service/compressed_body.rb +4 -1
  75. data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
  76. data/lib/html2rss/request_service/network_guard.rb +7 -5
  77. data/lib/html2rss/request_service/response.rb +1 -2
  78. data/lib/html2rss/request_service/strategy.rb +3 -8
  79. data/lib/html2rss/scoring/container_assessor.rb +3 -1
  80. data/lib/html2rss/scoring/engine.rb +8 -8
  81. data/lib/html2rss/scoring/observation.rb +5 -3
  82. data/lib/html2rss/scoring/ranked_segment.rb +3 -1
  83. data/lib/html2rss/scrape_target.rb +3 -1
  84. data/lib/html2rss/selector_candidates.rb +234 -0
  85. data/lib/html2rss/selectors/README.md +64 -0
  86. data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
  87. data/lib/html2rss/selectors/extractors/href.rb +15 -16
  88. data/lib/html2rss/selectors/extractors/html.rb +8 -9
  89. data/lib/html2rss/selectors/extractors/static.rb +12 -14
  90. data/lib/html2rss/selectors/extractors/text.rb +8 -9
  91. data/lib/html2rss/selectors/extractors.rb +33 -20
  92. data/lib/html2rss/selectors/item_env.rb +31 -0
  93. data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
  94. data/lib/html2rss/selectors/option_spec.rb +140 -0
  95. data/lib/html2rss/selectors/post_processors/base.rb +79 -31
  96. data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
  97. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
  98. data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
  99. data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
  100. data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
  101. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
  102. data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
  103. data/lib/html2rss/selectors/post_processors/template.rb +23 -19
  104. data/lib/html2rss/selectors/post_processors.rb +6 -5
  105. data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
  106. data/lib/html2rss/selectors/step_env.rb +24 -0
  107. data/lib/html2rss/selectors.rb +231 -142
  108. data/lib/html2rss/sst/attrs.rb +5 -3
  109. data/lib/html2rss/sst/document.rb +3 -1
  110. data/lib/html2rss/sst/index.rb +3 -1
  111. data/lib/html2rss/sst/node.rb +29 -10
  112. data/lib/html2rss/sst/normalizer.rb +12 -16
  113. data/lib/html2rss/sst/tags.rb +7 -7
  114. data/lib/html2rss/status.rb +13 -8
  115. data/lib/html2rss/surface_category.rb +14 -32
  116. data/lib/html2rss/syndication/parser.rb +2 -2
  117. data/lib/html2rss/test/enhance_audit.rb +1 -3
  118. data/lib/html2rss/test/policy.rb +106 -0
  119. data/lib/html2rss/test.rb +126 -173
  120. data/lib/html2rss/url.rb +25 -4
  121. data/lib/html2rss/version.rb +1 -1
  122. data/lib/html2rss.rb +21 -13
  123. data/lib/tasks/config_schema.rake +2 -2
  124. data/schema/html2rss-config.schema.json +11 -8
  125. metadata +15 -6
  126. data/lib/html2rss/selectors/item_scope.rb +0 -30
@@ -0,0 +1,234 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ ##
5
+ # Ranks Capture-derived selector evidence into deterministic best-first buckets.
6
+ #
7
+ # Owns collection into buckets, deduplication, ranking, and empty-bucket semantics.
8
+ # Does not emit the default +a[href]+ fallback — that remains Capture's YAML path only.
9
+ class SelectorCandidates # rubocop:disable Metrics/ClassLength -- ranking + field CSS stay co-located
10
+ # Empty candidate buckets (no discovered evidence).
11
+ EMPTY = { items: [].freeze, title: [].freeze, link: [].freeze, published: [].freeze }.freeze
12
+
13
+ # Ranking preference for items selector derivation kind (lower is better).
14
+ KIND_RANK = { shared_class: 0, unique_tag: 1, path: 2 }.freeze
15
+ private_constant :KIND_RANK
16
+
17
+ # Ranking preference for Segmenter strategy order (lower is better).
18
+ STRATEGY_RANK = { list: 0, cluster: 1, semantic: 2 }.freeze
19
+ private_constant :STRATEGY_RANK
20
+
21
+ # Minimum roots that must share a relative field selector.
22
+ DEFAULT_MIN_MATCHES = 2
23
+ private_constant :DEFAULT_MIN_MATCHES
24
+
25
+ # Class token that identifies a non-heading title node. Same pattern title_node matches.
26
+ TITLE_CLASS_PATTERN = /title|font-bold|font-semibold/
27
+ private_constant :TITLE_CLASS_PATTERN
28
+
29
+ class << self
30
+ ##
31
+ # @return [Hash{Symbol=>Array}] frozen empty buckets
32
+ def empty = EMPTY
33
+
34
+ ##
35
+ # Ranks items evidence and derives title/link/published field selectors.
36
+ #
37
+ # @param items_evidence [Array<Hash>] each with +:selector+, +:enhance+, +:strategy+,
38
+ # +:kind+, +:match_count+, and +:roots+ (SST nodes under the items selector)
39
+ # @param min_matches [Integer] minimum repeated field evidence across roots
40
+ # @return [Hash{Symbol=>Array<Hash>}] +:items+, +:title+, +:link+, +:published+
41
+ def call(items_evidence:, min_matches: DEFAULT_MIN_MATCHES)
42
+ new(items_evidence:, min_matches:).call
43
+ end
44
+ end
45
+
46
+ ##
47
+ # @param items_evidence [Array<Hash>]
48
+ # @param min_matches [Integer]
49
+ def initialize(items_evidence:, min_matches: DEFAULT_MIN_MATCHES)
50
+ @items_evidence = Array(items_evidence)
51
+ @min_matches = min_matches
52
+ end
53
+
54
+ ##
55
+ # @return [Hash{Symbol=>Array<Hash>}]
56
+ def call
57
+ items = rank_items(@items_evidence)
58
+ return EMPTY if items.empty?
59
+
60
+ roots = roots_for(items.first, @items_evidence)
61
+ {
62
+ items:,
63
+ title: field_bucket(roots, :title),
64
+ link: field_bucket(roots, :link),
65
+ published: field_bucket(roots, :published)
66
+ }
67
+ end
68
+
69
+ private
70
+
71
+ def rank_items(evidence)
72
+ evidence
73
+ .sort_by { |entry| item_rank_key(entry) }
74
+ .each_with_object([]) do |entry, ranked|
75
+ selector = entry[:selector].to_s
76
+ next if selector.empty? || ranked.any? { |c| c[:selector] == selector }
77
+
78
+ ranked << item_candidate(entry)
79
+ end
80
+ end
81
+
82
+ def item_candidate(entry)
83
+ {
84
+ selector: entry[:selector].to_s,
85
+ enhance: entry[:enhance],
86
+ sample: visible_sample(Array(entry[:roots]).first)
87
+ }
88
+ end
89
+
90
+ def item_rank_key(entry)
91
+ [
92
+ KIND_RANK.fetch(entry[:kind], 99),
93
+ STRATEGY_RANK.fetch(entry[:strategy], 99),
94
+ -entry[:match_count].to_i,
95
+ entry[:selector].to_s
96
+ ]
97
+ end
98
+
99
+ def roots_for(top_item, evidence)
100
+ match = evidence.find { |entry| entry[:selector].to_s == top_item[:selector] }
101
+ Array(match&.fetch(:roots, nil))
102
+ end
103
+
104
+ def field_bucket(roots, field)
105
+ usable = roots.grep(SST::Node)
106
+ return [] if usable.size < @min_matches
107
+
108
+ rank_field_tallies(count_field_selectors(usable, field))
109
+ end
110
+
111
+ def count_field_selectors(roots, field)
112
+ roots.each_with_object({}) do |root, tallies|
113
+ node = field_node(root, field)
114
+ selector = relative_field_selector(root, field)
115
+ next unless selector
116
+
117
+ tallies[selector] ||= { count: 0, sample: visible_sample(node) }
118
+ tallies[selector][:count] += 1
119
+ end
120
+ end
121
+
122
+ def rank_field_tallies(tallies)
123
+ tallies
124
+ .select { |_selector, data| data[:count] >= @min_matches }
125
+ .sort_by { |selector, data| [-data[:count], selector.length, selector] }
126
+ .map { |selector, data| { selector:, sample: data[:sample] } }
127
+ end
128
+
129
+ def visible_sample(node)
130
+ node&.visible_text.to_s.gsub(/\s+/, ' ').strip
131
+ end
132
+
133
+ def relative_field_selector(root, field)
134
+ node = field_node(root, field)
135
+ token = identifying_title_class(node) if field == :title
136
+ relative_css(root, node, class_token: token)
137
+ end
138
+
139
+ def field_node(root, field)
140
+ case field
141
+ when :title then title_node(root)
142
+ when :link then link_node(root)
143
+ when :published then published_node(root)
144
+ end
145
+ end
146
+
147
+ def title_node(root)
148
+ headings = root.find_all(&:heading?)
149
+ return best_heading(headings) if headings.any?
150
+
151
+ root.find do |node|
152
+ next if node.equal?(root)
153
+
154
+ identifying_title_class(node) && !node.visible_text.to_s.strip.empty?
155
+ end
156
+ end
157
+
158
+ def best_heading(headings)
159
+ min_name = headings.map { |node| node.name.to_s }.min
160
+ headings.select { |node| node.name.to_s == min_name }
161
+ .max_by { |node| node.visible_text.to_s.size }
162
+ end
163
+
164
+ def link_node(root)
165
+ return if root.link?
166
+
167
+ root.find(&:link?)
168
+ end
169
+
170
+ def published_node(root)
171
+ root.find do |node|
172
+ next if node.equal?(root)
173
+
174
+ node.name == :time || !node.attrs.datetime.to_s.empty?
175
+ end
176
+ end
177
+
178
+ # First class that title_node used to accept this node, not the lexicographically smallest token.
179
+ def identifying_title_class(node)
180
+ return if node.nil? || node.heading?
181
+
182
+ node.attrs.class_names.find { |name| name.match?(TITLE_CLASS_PATTERN) }
183
+ end
184
+
185
+ def relative_css(root, node, class_token:)
186
+ return if node.nil? || node.equal?(root)
187
+ return "#{node.name}.#{css_escape_ident(class_token)}" if class_token
188
+
189
+ relative_path_css(root, node)
190
+ end
191
+
192
+ # CSSOM "serialize an identifier" (CSS.escape). Class tokens are untrusted HTML.
193
+ def css_escape_ident(token)
194
+ return '\\-' if token == '-'
195
+
196
+ token.each_char.with_index.map { |char, index| css_escaped_char(char, index, token) }.join
197
+ end
198
+
199
+ def css_escaped_char(char, index, token)
200
+ code = char.ord
201
+ return "\uFFFD" if code.zero?
202
+ return "\\#{code.to_s(16)} " if css_hex_escape?(code, index, token)
203
+ return char if css_ident_literal?(code)
204
+
205
+ "\\#{char}"
206
+ end
207
+
208
+ def css_hex_escape?(code, index, token)
209
+ code.between?(1, 0x1F) || code == 0x7F ||
210
+ (index.zero? && code.between?(0x30, 0x39)) ||
211
+ (index == 1 && code.between?(0x30, 0x39) && token.start_with?('-'))
212
+ end
213
+
214
+ def css_ident_literal?(code)
215
+ code >= 0x80 || code == 0x2D || code == 0x5F ||
216
+ code.between?(0x30, 0x39) || code.between?(0x41, 0x5A) || code.between?(0x61, 0x7A)
217
+ end
218
+
219
+ def relative_path_css(root, node)
220
+ root_parts = path_parts(root)
221
+ node_parts = path_parts(node)
222
+ return unless node_parts[0, root_parts.length] == root_parts
223
+
224
+ rel = node_parts[root_parts.length..]
225
+ return if rel.nil? || rel.empty?
226
+
227
+ rel.join(' > ')
228
+ end
229
+
230
+ def path_parts(node)
231
+ node.tag_path.to_s.split('/').reject(&:empty?)
232
+ end
233
+ end
234
+ end
@@ -0,0 +1,64 @@
1
+ # Selectors
2
+
3
+ How html2rss extracts feed item fields from a page when a config declares CSS `selectors:`.
4
+
5
+ ## What and when
6
+
7
+ `Html2rss::Selectors` is the **orchestrator** for the traditional selector path: match item nodes, dispatch each configured field through extractors and optional post-processors, then build `Article` hashes. Prefer this when you know the list markup. Prefer `auto_source:` when you want article guessing without hand-written selectors (see {file:lib/html2rss/auto_source/README.md}).
8
+
9
+ YAML / MCP wire vocabulary is unchanged (`selectors`, `extractor`, `post_process`). Ruby type names below are the internal API.
10
+
11
+ ## Live flow
12
+
13
+ 1. **Parse** — HTML stays on `Response#parsed_body`. JSON responses are converted once via `JsonXml` into an HTML5 fragment so CSS selectors still run.
14
+ 2. **Items** — `items.selector` finds each card/node. Omitted, blank, and `Selectors::DEFAULT_ITEMS_SELECTOR` expand to `Html::Navigator::MAIN_ANCHOR_SELECTOR` (permalink anchors). Any other selector, including `a` and `.card`, is used as written. `Config.prepare_defaults` fills that default, with `enhance: true`, only when `selectors:` is present. `enhance: true` fills missing fields via `Html::ArticleExtractor` (list-card enrichment). Enhancement is skipped when title, url, description, published_at, enclosures, and categories are already populated.
15
+ 3. **Field dispatch** (private on `Selectors`) — for each selectable key (`title`, `url`, `enclosure`, `categories`, …):
16
+ - Regular fields → `Extractors.call` → optional `PostProcessors.call` chain
17
+ - Special keys → enclosure wrap, guid fan-out, multi-node categories
18
+ 4. **Article** — values land on `Html2rss::Article` (`enclosure` YAML → `enclosures` on the Article).
19
+
20
+ ```text
21
+ Selectors (orchestrator)
22
+ └─ ItemEnv
23
+ └─ field dispatch (private)
24
+ ├─ Extractors::*#call(**config, xml:) + OPTIONS: [OptionSpec]
25
+ └─ PostProcessors::*#call + StepEnv(base_url:, time_zone:, step_config:, item_env:)
26
+ ```
27
+
28
+ ## The Three Layers of #call
29
+
30
+ | Layer | Entrypoint | Role | Inputs / Outputs |
31
+ | --- | --- | --- | --- |
32
+ | **Registry dispatcher** | `Extractors.call(config, xml)` | Resolves strategy from `NAME_TO_CLASS`, instantiates with `**config`, executes `#call` | `(config Hash, xml Node)` → extracted value |
33
+ | **Registry dispatcher** | `PostProcessors.call(name, value, context)` | Resolves strategy from `NAME_TO_CLASS`, passes `StepEnv`, executes `#call` | `(name, value, StepEnv)` → transformed value |
34
+ | **Strategy execution** | `Extractors::*#call` | Polymorphic instance execution on extractor strategies | Keyword config on `initialize`, returns extracted Object |
35
+ | **Strategy execution** | `PostProcessors::*#call` | Polymorphic instance execution on post-processor strategies | Reads `@value` and `@context`, returns transformed Object |
36
+ | **Data converter** | `JsonXml.call(object)` | Internal helper converting JSON object to XML string fragment | `(Hash / Array)` → `String` XML fragment |
37
+ | **Standalone sanitizer** | `SanitizeHtml.call(html, url)` | Cached helper for description/standalone sanitization | `(html, url)` → sanitized HTML String |
38
+
39
+ ## Extractor vs PostProcessor
40
+
41
+ | Role | Registry | Verb | Config |
42
+ | --- | --- | --- | --- |
43
+ | Extractor | `Extractors::NAME_TO_CLASS` | `#call` / `Extractors.call` | YAML `extractor:` + strategy `OPTIONS` |
44
+ | Post-processor | `PostProcessors::NAME_TO_CLASS` | `#call` / `PostProcessors.call` | YAML `post_process:` steps with `name:` + `OPTIONS` |
45
+
46
+ `Extractors::Attribute` is the HTML-attribute strategy — not the field-dispatch layer (that is private on `Selectors`).
47
+
48
+ ## ItemEnv and StepEnv
49
+
50
+ - **`ItemEnv`** — per-item extraction environment: node, `base_url`, scraper, `time_zone`. Nested selects reuse one env via `#select`.
51
+ - **`StepEnv`** — post-processor invocation bag: `step_config` (Hash), `base_url`, `time_zone`, `item_env`. Built by `ItemEnv#context_for(step_config:)`. Distinct from `OptionSpec` (static strategy introspection). Channel URL on Test / MCP is a different field.
52
+
53
+ ## OptionSpec vs SchemaExport
54
+
55
+ - **`OptionSpec`** — introspection & validation SoT (`for(klass)`, `expectation_for`, `valid_type?`, `error_message`, `json_type`) with **Ruby-typed** `type:` / `required:`. Strategies own `OPTIONS: [OptionSpec, …]`.
56
+ - **`SchemaExport`** — JSON Schema adapter only (`for_extractor`, `for_post_processor`), consuming `OptionSpec#json_type`. `Config::IssueMapper` maps Ruby expectations → JSON via `OptionSpec.json_type_for` so ValidationIssue `expected` wire shape stays stable.
57
+
58
+ ## Constraints
59
+
60
+ - Class / YAML name stays `Selectors`.
61
+ - Strategy verb is `#call`.
62
+ - `gsub` rejects a `pattern` longer than 256 characters as written, or whose star-height is greater than 1; `Html2rss.validate` and extraction share that compile.
63
+
64
+ See also {file:CONTEXT CONTEXT.md} for registry ownership and {Html2rss::Config::SelectorsValidator} for Dry validation of the selectors hash.
@@ -24,13 +24,12 @@ module Html2rss
24
24
  # In case you're extracting a date or a time, consider parsing it
25
25
  # during post processing with {PostProcessors::ParseTime}.
26
26
  class Attribute
27
- # Config-facing option types (excluding shared selector fields supplied at runtime).
28
- OPTION_TYPES = { attribute: String }.freeze
29
-
30
- # The available options for the attribute extractor.
31
- Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
27
+ # Config-facing options contract (validator introspection; +selector+ is runtime-only).
28
+ OPTIONS = [
29
+ OptionSpec.new(name: :attribute, type: String)
30
+ ].freeze
32
31
 
33
- # JSON Schema description exported via +schema_doc+.
32
+ # JSON Schema description exported via +schema_export+.
34
33
  DESCRIPTION = 'Return the value of an HTML attribute on the selected element. ' \
35
34
  'Requires sibling selector option `attribute` (attribute name).'
36
35
 
@@ -40,26 +39,25 @@ module Html2rss
40
39
  ].freeze
41
40
 
42
41
  # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
43
- def self.schema_doc = SchemaDoc.for_extractor(name: :attribute, klass: self)
42
+ def self.schema_export = SchemaExport.for_extractor(name: :attribute, klass: self)
44
43
 
45
44
  ##
46
45
  # Initializes the Attribute extractor.
47
46
  #
48
47
  # @param xml [Nokogiri::XML::Element]
49
- # @param options [Options]
50
- # @option options [String] :selector CSS selector used to find the element
51
- # @option options [String] :attribute attribute name to extract from the selected element
52
- def initialize(xml, options)
53
- @options = options
54
- @element = Extractors.element(xml, options.selector)
48
+ # @param attribute [String, nil] attribute name to extract from the selected element
49
+ # @param selector [String, nil] CSS selector used to find the element
50
+ def initialize(xml, attribute: nil, selector: nil, **)
51
+ @attribute = attribute
52
+ @element = Extractors.element(xml, selector, first: true)
55
53
  end
56
54
 
57
55
  ##
58
56
  # Retrieves and returns the attribute's value as a string.
59
57
  #
60
58
  # @return [String] The value of the attribute.
61
- def get
62
- @element.attr(@options.attribute).to_s
59
+ def call
60
+ @element&.[](@attribute).to_s
63
61
  end
64
62
  end
65
63
  end
@@ -6,7 +6,7 @@ module Html2rss
6
6
  ##
7
7
  # Returns the value of the +href+ attribute.
8
8
  # It always returns absolute URLs. If the extracted +href+ value is a
9
- # relative URL, it prepends the channel's URL.
9
+ # relative URL, it prepends the page base URL.
10
10
  #
11
11
  # Imagine this +a+ HTML element with a +href+ attribute:
12
12
  #
@@ -24,12 +24,12 @@ module Html2rss
24
24
  # Would return:
25
25
  # 'http://blog-without-a-feed.example.com/posts/latest-findings'
26
26
  class Href
27
- # The available options for the href (attribute) extractor.
28
- Options = Struct.new('HrefOptions', :selector, :channel, keyword_init: true)
27
+ # Config-facing options contract (none).
28
+ OPTIONS = [].freeze
29
29
 
30
- # JSON Schema description exported via +schema_doc+.
30
+ # JSON Schema description exported via +schema_export+.
31
31
  DESCRIPTION = 'Return the absolute URL from the selected element\'s `href` attribute ' \
32
- '(relative hrefs are resolved against the channel URL).'
32
+ '(relative hrefs are resolved against the page base URL).'
33
33
 
34
34
  # Example extractor name values for JSON Schema +examples+.
35
35
  EXAMPLES = [
@@ -37,29 +37,28 @@ module Html2rss
37
37
  ].freeze
38
38
 
39
39
  # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
40
- def self.schema_doc = SchemaDoc.for_extractor(name: :href, klass: self)
40
+ def self.schema_export = SchemaExport.for_extractor(name: :href, klass: self)
41
41
 
42
42
  ##
43
43
  # Initializes the Href extractor.
44
44
  #
45
45
  # @param xml [Nokogiri::XML::Element]
46
- # @param options [Options]
47
- # @option options [String] :selector CSS selector used to find the link element
48
- # @option options [Hash{Symbol => Object}] :channel channel configuration, including :url
49
- def initialize(xml, options)
50
- @options = options
51
- @element = Extractors.element(xml, options.selector)
52
- @href = @element.attr('href').to_s
46
+ # @param base_url [String, Html2rss::Url, nil] page base URL for relative hrefs
47
+ # @param selector [String, nil] CSS selector used to find the link element
48
+ def initialize(xml, base_url: nil, selector: nil, **)
49
+ @base_url = base_url
50
+ @element = Extractors.element(xml, selector, first: true)
51
+ @href = @element&.[]('href').to_s
53
52
  end
54
53
 
55
54
  ##
56
55
  # Retrieves and returns the normalized absolute URL.
57
56
  #
58
- # @return [String] The absolute URL.
59
- def get
57
+ # @return [Html2rss::Url, nil] The absolute URL.
58
+ def call
60
59
  return nil unless @href
61
60
 
62
- Url.from_relative(@href, @options.channel[:url])
61
+ Url.from_relative(@href, @base_url)
63
62
  end
64
63
  end
65
64
  end
@@ -23,10 +23,10 @@ module Html2rss
23
23
  # Always ensure to sanitize the HTML during post-processing with
24
24
  # {PostProcessors::SanitizeHtml}.
25
25
  class Html
26
- # The available options for the html extractor.
27
- Options = Struct.new('HtmlOptions', :selector, keyword_init: true)
26
+ # Config-facing options contract (none).
27
+ OPTIONS = [].freeze
28
28
 
29
- # JSON Schema description exported via +schema_doc+.
29
+ # JSON Schema description exported via +schema_export+.
30
30
  DESCRIPTION = 'Return the outer HTML of the selected element. ' \
31
31
  'Sanitize during post-processing (e.g. `sanitize_html`).'
32
32
 
@@ -36,23 +36,22 @@ module Html2rss
36
36
  ].freeze
37
37
 
38
38
  # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
39
- def self.schema_doc = SchemaDoc.for_extractor(name: :html, klass: self)
39
+ def self.schema_export = SchemaExport.for_extractor(name: :html, klass: self)
40
40
 
41
41
  ##
42
42
  # Initializes the Html extractor.
43
43
  #
44
44
  # @param xml [Nokogiri::XML::Element]
45
- # @param options [Options]
46
- # @option options [String] :selector CSS selector used to find the element
47
- def initialize(xml, options)
48
- @element = Extractors.element(xml, options.selector)
45
+ # @param selector [String, nil] CSS selector used to find the element
46
+ def initialize(xml, selector: nil, **)
47
+ @element = Extractors.element(xml, selector)
49
48
  end
50
49
 
51
50
  ##
52
51
  # Retrieves and returns the HTML content of the element.
53
52
  #
54
53
  # @return [String] The HTML content.
55
- def get
54
+ def call
56
55
  @element.to_s
57
56
  end
58
57
  end
@@ -16,13 +16,12 @@ module Html2rss
16
16
  # Would return:
17
17
  # 'Foobar'
18
18
  class Static
19
- # Config-facing option types (excluding shared selector fields supplied at runtime).
20
- OPTION_TYPES = { static: String }.freeze
21
-
22
- # The available option for the static extractor.
23
- Options = Struct.new('StaticOptions', :static, keyword_init: true)
19
+ # Config-facing options contract (validator introspection).
20
+ OPTIONS = [
21
+ OptionSpec.new(name: :static, type: String)
22
+ ].freeze
24
23
 
25
- # JSON Schema description exported via +schema_doc+.
24
+ # JSON Schema description exported via +schema_export+.
26
25
  DESCRIPTION = 'Return a fixed value from sibling selector option `static` (no DOM read).'
27
26
 
28
27
  # Example extractor name values for JSON Schema +examples+.
@@ -31,24 +30,23 @@ module Html2rss
31
30
  ].freeze
32
31
 
33
32
  # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
34
- def self.schema_doc = SchemaDoc.for_extractor(name: :static, klass: self)
33
+ def self.schema_export = SchemaExport.for_extractor(name: :static, klass: self)
35
34
 
36
35
  ##
37
36
  # Initializes the Static extractor.
38
37
  #
39
38
  # @param _xml [nil, Nokogiri::XML::Element] Unused parameter for compatibility with other extractors.
40
- # @param options [Options] Options containing the static value.
41
- # @option options [String, Symbol] :static static value returned by this extractor
42
- def initialize(_xml, options)
43
- @options = options
39
+ # @param static [String, Symbol, nil] static value returned by this extractor
40
+ def initialize(_xml = nil, static: nil, **)
41
+ @static = static
44
42
  end
45
43
 
46
44
  ##
47
45
  # Retrieves and returns the static value.
48
46
  #
49
- # @return [String, Symbol] The static value provided in options.
50
- def get
51
- @options.static
47
+ # @return [String, Symbol, nil] The static value.
48
+ def call
49
+ @static
52
50
  end
53
51
  end
54
52
  end
@@ -21,10 +21,10 @@ module Html2rss
21
21
  # Would return:
22
22
  # 'Lorem ipsum dolor ...'
23
23
  class Text
24
- # The available options for the text extractor.
25
- Options = Struct.new('TextOptions', :selector, keyword_init: true)
24
+ # Config-facing options contract (none).
25
+ OPTIONS = [].freeze
26
26
 
27
- # JSON Schema description exported via +schema_doc+.
27
+ # JSON Schema description exported via +schema_export+.
28
28
  DESCRIPTION = 'Return collapsed visible text of the selected element (default extractor).'
29
29
 
30
30
  # Example extractor name values for JSON Schema +examples+.
@@ -33,23 +33,22 @@ module Html2rss
33
33
  ].freeze
34
34
 
35
35
  # @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
36
- def self.schema_doc = SchemaDoc.for_extractor(name: :text, klass: self)
36
+ def self.schema_export = SchemaExport.for_extractor(name: :text, klass: self)
37
37
 
38
38
  ##
39
39
  # Initializes the Text extractor.
40
40
  #
41
41
  # @param xml [Nokogiri::XML::Element]
42
- # @param options [Options]
43
- # @option options [String] :selector CSS selector used to find the element
44
- def initialize(xml, options)
45
- @element = Extractors.element(xml, options.selector)
42
+ # @param selector [String, nil] CSS selector used to find the element
43
+ def initialize(xml, selector: nil, **)
44
+ @element = Extractors.element(xml, selector)
46
45
  end
47
46
 
48
47
  ##
49
48
  # Retrieves and returns the text content of the element.
50
49
  #
51
50
  # @return [String] The text content.
52
- def get
51
+ def call
53
52
  @element.text.to_s.strip.gsub(/\s+/, ' ')
54
53
  end
55
54
  end
@@ -17,35 +17,48 @@ module Html2rss
17
17
  text: Text
18
18
  }.freeze
19
19
 
20
- ##
21
- # Maps the extractor class to its corresponding options class.
22
- ITEM_OPTION_CLASSES = Hash.new do |hash, klass|
23
- hash[klass] = klass.const_get(:Options)
24
- end
25
-
26
20
  # Extractor used when none is explicitly configured.
27
21
  DEFAULT_EXTRACTOR = :text
28
-
29
22
  class << self
30
23
  ##
31
24
  # Retrieves an element from Nokogiri XML based on the selector.
32
25
  #
33
- # @param xml [Nokogiri::XML::Document]
26
+ # Attribute-style extractors pass +first: true+ so Nokogiri can use +at_css+
27
+ # when only the first match is consumed. Text/HTML keep +css+ so every
28
+ # match is concatenated. Category collection also uses the default.
29
+ #
30
+ # @param xml [Nokogiri::XML::Document, Nokogiri::XML::Node]
34
31
  # @param selector [String, nil]
35
- # @return [Nokogiri::XML::ElementSet] selected XML elements
36
- def element(xml, selector)
37
- selector ? xml.css(selector) : xml
38
- end
32
+ # @param first [Boolean] when true, return the first match via +at_css+
33
+ # @return [Nokogiri::XML::Node, Nokogiri::XML::NodeSet, nil] selected XML elements
34
+ def element(xml, selector, first: false)
35
+ return xml unless selector
39
36
 
40
- # @param attribute_options [Hash{Symbol => Object}]
41
- # Should contain at least `:extractor` (the name) and required options for that extractor.
42
- # @param xml [Nokogiri::XML::Document]
43
- # @return [Object] instance of the specified item extractor class
44
- def get(attribute_options, xml)
45
- extractor_class = NAME_TO_CLASS[attribute_options[:extractor]&.to_sym || DEFAULT_EXTRACTOR]
46
- options = ITEM_OPTION_CLASSES[extractor_class].new(attribute_options.slice(*extractor_class::Options.members))
37
+ first ? xml.at_css(selector) : xml.css(selector)
38
+ end
47
39
 
48
- extractor_class.new(xml, options).get
40
+ ##
41
+ # Registry dispatcher: resolves the extractor strategy for +config+,
42
+ # instantiates it with options, and executes its +#call+.
43
+ #
44
+ # +base_url+ and +selector+ can be supplied without merging a new Hash
45
+ # into the (static) field config on every item.
46
+ #
47
+ # @param config [Hash{Symbol => Object}]
48
+ # Should contain at least `:extractor` (the name) and options for that extractor.
49
+ # @param xml [Nokogiri::XML::Node, Nokogiri::XML::NodeSet]
50
+ # @param base_url [String, Html2rss::Url, nil] page base URL for relative hrefs
51
+ # @param selector [String, nil] CSS selector override (pass +nil+ to use +xml+ as-is)
52
+ # @return [Object] extracted value from the strategy instance
53
+ def call(config, xml, base_url: nil, selector: config[:selector])
54
+ extractor_class = NAME_TO_CLASS[config[:extractor]&.to_sym || DEFAULT_EXTRACTOR]
55
+ extractor_class.new(
56
+ xml,
57
+ selector:,
58
+ attribute: config[:attribute],
59
+ static: config[:static],
60
+ base_url: base_url || config[:base_url]
61
+ ).call
49
62
  end
50
63
  end
51
64
  end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ class Selectors
5
+ ItemEnv = Data.define(:item, :base_url, :scraper, :time_zone)
6
+
7
+ ##
8
+ # Per-item extraction environment: owns the article node, page +base_url+, and
9
+ # +time_zone+ for one extraction pass.
10
+ #
11
+ # Distinct from {StepEnv}, which is the post-processor invocation bag (+step_config+).
12
+ class ItemEnv
13
+ ##
14
+ # Selects an attribute using this env's item and base_url.
15
+ #
16
+ # @param name [Symbol, String]
17
+ # @return [Object, Array<Object>]
18
+ def select(name) = scraper.select_in_scope(name, self)
19
+
20
+ ##
21
+ # Builds a post-processor {StepEnv} carrying this env for nested selects.
22
+ #
23
+ # @param step_config [Hash] post-processor configuration from the selector config
24
+ # @option step_config [String] :name post-processor name
25
+ # @return [StepEnv]
26
+ def context_for(step_config:)
27
+ StepEnv.new(step_config:, item_env: self)
28
+ end
29
+ end
30
+ end
31
+ end