html2rss 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/html2rss/article/deduplicator.rb +2 -4
- data/lib/html2rss/article.rb +16 -39
- data/lib/html2rss/auto_source/README.md +1 -1
- data/lib/html2rss/auto_source/cleanup.rb +15 -6
- data/lib/html2rss/auto_source/scraper/html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
- data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
- data/lib/html2rss/auto_source/scraper.rb +4 -14
- data/lib/html2rss/auto_source/segment.rb +7 -4
- data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
- data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
- data/lib/html2rss/auto_source/segmenter.rb +5 -5
- data/lib/html2rss/auto_source.rb +2 -2
- data/lib/html2rss/batch.rb +3 -1
- data/lib/html2rss/capture/README.md +6 -4
- data/lib/html2rss/capture.rb +67 -35
- data/lib/html2rss/channel.rb +3 -3
- data/lib/html2rss/cli/probe_view.rb +5 -3
- data/lib/html2rss/cli/render.rb +6 -5
- data/lib/html2rss/cli/validate.rb +28 -27
- data/lib/html2rss/cli.rb +1 -1
- data/lib/html2rss/config/issue_mapper.rb +108 -0
- data/lib/html2rss/config/request_headers.rb +2 -3
- data/lib/html2rss/config/schema.rb +16 -1
- data/lib/html2rss/config/selectors_validator.rb +92 -70
- data/lib/html2rss/config/validation_issue.rb +60 -0
- data/lib/html2rss/config/validation_report.rb +55 -0
- data/lib/html2rss/config/validator.rb +10 -6
- data/lib/html2rss/config.rb +35 -113
- data/lib/html2rss/defaults.rb +42 -30
- data/lib/html2rss/doctor/botasaurus.rb +6 -4
- data/lib/html2rss/doctor.rb +8 -0
- data/lib/html2rss/error.rb +1 -1
- data/lib/html2rss/feed_builder/rss.rb +9 -14
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
- data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
- data/lib/html2rss/feed_resolution/diag.rb +3 -1
- data/lib/html2rss/feed_resolution/options.rb +3 -1
- data/lib/html2rss/feed_resolution/policy.rb +2 -2
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +2 -4
- data/lib/html2rss/html/feed_link.rb +3 -1
- data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
- data/lib/html2rss/html/navigator.rb +10 -10
- data/lib/html2rss/html/rendering/description_builder.rb +1 -1
- data/lib/html2rss/html/sst_article_extractor.rb +8 -5
- data/lib/html2rss/link_destination/destination_facts.rb +4 -2
- data/lib/html2rss/link_destination/noise_policy.rb +10 -1
- data/lib/html2rss/link_destination/path_classifier.rb +45 -38
- data/lib/html2rss/mcp/README.md +2 -0
- data/lib/html2rss/mcp/config_argument.rb +3 -1
- data/lib/html2rss/mcp/contract.rb +6 -20
- data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
- data/lib/html2rss/mcp/outcome.rb +36 -43
- data/lib/html2rss/mcp/server/tools.rb +111 -131
- data/lib/html2rss/mcp/server.rb +26 -18
- data/lib/html2rss/page_recon/README.md +5 -5
- data/lib/html2rss/page_recon/diagnostics.rb +71 -81
- data/lib/html2rss/page_recon.rb +9 -5
- data/lib/html2rss/recon.rb +19 -30
- data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
- data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
- data/lib/html2rss/request_service/compressed_body.rb +4 -1
- data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
- data/lib/html2rss/request_service/network_guard.rb +7 -5
- data/lib/html2rss/request_service/response.rb +1 -2
- data/lib/html2rss/request_service/strategy.rb +3 -8
- data/lib/html2rss/scoring/container_assessor.rb +3 -1
- data/lib/html2rss/scoring/engine.rb +8 -8
- data/lib/html2rss/scoring/observation.rb +5 -3
- data/lib/html2rss/scoring/ranked_segment.rb +3 -1
- data/lib/html2rss/scrape_target.rb +3 -1
- data/lib/html2rss/selector_candidates.rb +234 -0
- data/lib/html2rss/selectors/README.md +64 -0
- data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
- data/lib/html2rss/selectors/extractors/href.rb +15 -16
- data/lib/html2rss/selectors/extractors/html.rb +8 -9
- data/lib/html2rss/selectors/extractors/static.rb +12 -14
- data/lib/html2rss/selectors/extractors/text.rb +8 -9
- data/lib/html2rss/selectors/extractors.rb +33 -20
- data/lib/html2rss/selectors/item_env.rb +31 -0
- data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
- data/lib/html2rss/selectors/option_spec.rb +140 -0
- data/lib/html2rss/selectors/post_processors/base.rb +79 -31
- data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
- data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
- data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
- data/lib/html2rss/selectors/post_processors/template.rb +23 -19
- data/lib/html2rss/selectors/post_processors.rb +6 -5
- data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
- data/lib/html2rss/selectors/step_env.rb +24 -0
- data/lib/html2rss/selectors.rb +231 -142
- data/lib/html2rss/sst/attrs.rb +5 -3
- data/lib/html2rss/sst/document.rb +3 -1
- data/lib/html2rss/sst/index.rb +3 -1
- data/lib/html2rss/sst/node.rb +29 -10
- data/lib/html2rss/sst/normalizer.rb +12 -16
- data/lib/html2rss/sst/tags.rb +7 -7
- data/lib/html2rss/status.rb +13 -8
- data/lib/html2rss/surface_category.rb +14 -32
- data/lib/html2rss/syndication/parser.rb +2 -2
- data/lib/html2rss/test/enhance_audit.rb +1 -3
- data/lib/html2rss/test/policy.rb +106 -0
- data/lib/html2rss/test.rb +126 -173
- data/lib/html2rss/url.rb +25 -4
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +21 -13
- data/lib/tasks/config_schema.rake +2 -2
- data/schema/html2rss-config.schema.json +11 -8
- metadata +15 -6
- data/lib/html2rss/selectors/item_scope.rb +0 -30
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
##
|
|
5
|
+
# Ranks Capture-derived selector evidence into deterministic best-first buckets.
|
|
6
|
+
#
|
|
7
|
+
# Owns collection into buckets, deduplication, ranking, and empty-bucket semantics.
|
|
8
|
+
# Does not emit the default +a[href]+ fallback — that remains Capture's YAML path only.
|
|
9
|
+
class SelectorCandidates # rubocop:disable Metrics/ClassLength -- ranking + field CSS stay co-located
|
|
10
|
+
# Empty candidate buckets (no discovered evidence).
|
|
11
|
+
EMPTY = { items: [].freeze, title: [].freeze, link: [].freeze, published: [].freeze }.freeze
|
|
12
|
+
|
|
13
|
+
# Ranking preference for items selector derivation kind (lower is better).
|
|
14
|
+
KIND_RANK = { shared_class: 0, unique_tag: 1, path: 2 }.freeze
|
|
15
|
+
private_constant :KIND_RANK
|
|
16
|
+
|
|
17
|
+
# Ranking preference for Segmenter strategy order (lower is better).
|
|
18
|
+
STRATEGY_RANK = { list: 0, cluster: 1, semantic: 2 }.freeze
|
|
19
|
+
private_constant :STRATEGY_RANK
|
|
20
|
+
|
|
21
|
+
# Minimum roots that must share a relative field selector.
|
|
22
|
+
DEFAULT_MIN_MATCHES = 2
|
|
23
|
+
private_constant :DEFAULT_MIN_MATCHES
|
|
24
|
+
|
|
25
|
+
# Class token that identifies a non-heading title node. Same pattern title_node matches.
|
|
26
|
+
TITLE_CLASS_PATTERN = /title|font-bold|font-semibold/
|
|
27
|
+
private_constant :TITLE_CLASS_PATTERN
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
##
|
|
31
|
+
# @return [Hash{Symbol=>Array}] frozen empty buckets
|
|
32
|
+
def empty = EMPTY
|
|
33
|
+
|
|
34
|
+
##
|
|
35
|
+
# Ranks items evidence and derives title/link/published field selectors.
|
|
36
|
+
#
|
|
37
|
+
# @param items_evidence [Array<Hash>] each with +:selector+, +:enhance+, +:strategy+,
|
|
38
|
+
# +:kind+, +:match_count+, and +:roots+ (SST nodes under the items selector)
|
|
39
|
+
# @param min_matches [Integer] minimum repeated field evidence across roots
|
|
40
|
+
# @return [Hash{Symbol=>Array<Hash>}] +:items+, +:title+, +:link+, +:published+
|
|
41
|
+
def call(items_evidence:, min_matches: DEFAULT_MIN_MATCHES)
|
|
42
|
+
new(items_evidence:, min_matches:).call
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
##
|
|
47
|
+
# @param items_evidence [Array<Hash>]
|
|
48
|
+
# @param min_matches [Integer]
|
|
49
|
+
def initialize(items_evidence:, min_matches: DEFAULT_MIN_MATCHES)
|
|
50
|
+
@items_evidence = Array(items_evidence)
|
|
51
|
+
@min_matches = min_matches
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
##
|
|
55
|
+
# @return [Hash{Symbol=>Array<Hash>}]
|
|
56
|
+
def call
|
|
57
|
+
items = rank_items(@items_evidence)
|
|
58
|
+
return EMPTY if items.empty?
|
|
59
|
+
|
|
60
|
+
roots = roots_for(items.first, @items_evidence)
|
|
61
|
+
{
|
|
62
|
+
items:,
|
|
63
|
+
title: field_bucket(roots, :title),
|
|
64
|
+
link: field_bucket(roots, :link),
|
|
65
|
+
published: field_bucket(roots, :published)
|
|
66
|
+
}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def rank_items(evidence)
|
|
72
|
+
evidence
|
|
73
|
+
.sort_by { |entry| item_rank_key(entry) }
|
|
74
|
+
.each_with_object([]) do |entry, ranked|
|
|
75
|
+
selector = entry[:selector].to_s
|
|
76
|
+
next if selector.empty? || ranked.any? { |c| c[:selector] == selector }
|
|
77
|
+
|
|
78
|
+
ranked << item_candidate(entry)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def item_candidate(entry)
|
|
83
|
+
{
|
|
84
|
+
selector: entry[:selector].to_s,
|
|
85
|
+
enhance: entry[:enhance],
|
|
86
|
+
sample: visible_sample(Array(entry[:roots]).first)
|
|
87
|
+
}
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def item_rank_key(entry)
|
|
91
|
+
[
|
|
92
|
+
KIND_RANK.fetch(entry[:kind], 99),
|
|
93
|
+
STRATEGY_RANK.fetch(entry[:strategy], 99),
|
|
94
|
+
-entry[:match_count].to_i,
|
|
95
|
+
entry[:selector].to_s
|
|
96
|
+
]
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def roots_for(top_item, evidence)
|
|
100
|
+
match = evidence.find { |entry| entry[:selector].to_s == top_item[:selector] }
|
|
101
|
+
Array(match&.fetch(:roots, nil))
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def field_bucket(roots, field)
|
|
105
|
+
usable = roots.grep(SST::Node)
|
|
106
|
+
return [] if usable.size < @min_matches
|
|
107
|
+
|
|
108
|
+
rank_field_tallies(count_field_selectors(usable, field))
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def count_field_selectors(roots, field)
|
|
112
|
+
roots.each_with_object({}) do |root, tallies|
|
|
113
|
+
node = field_node(root, field)
|
|
114
|
+
selector = relative_field_selector(root, field)
|
|
115
|
+
next unless selector
|
|
116
|
+
|
|
117
|
+
tallies[selector] ||= { count: 0, sample: visible_sample(node) }
|
|
118
|
+
tallies[selector][:count] += 1
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def rank_field_tallies(tallies)
|
|
123
|
+
tallies
|
|
124
|
+
.select { |_selector, data| data[:count] >= @min_matches }
|
|
125
|
+
.sort_by { |selector, data| [-data[:count], selector.length, selector] }
|
|
126
|
+
.map { |selector, data| { selector:, sample: data[:sample] } }
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def visible_sample(node)
|
|
130
|
+
node&.visible_text.to_s.gsub(/\s+/, ' ').strip
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def relative_field_selector(root, field)
|
|
134
|
+
node = field_node(root, field)
|
|
135
|
+
token = identifying_title_class(node) if field == :title
|
|
136
|
+
relative_css(root, node, class_token: token)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def field_node(root, field)
|
|
140
|
+
case field
|
|
141
|
+
when :title then title_node(root)
|
|
142
|
+
when :link then link_node(root)
|
|
143
|
+
when :published then published_node(root)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def title_node(root)
|
|
148
|
+
headings = root.find_all(&:heading?)
|
|
149
|
+
return best_heading(headings) if headings.any?
|
|
150
|
+
|
|
151
|
+
root.find do |node|
|
|
152
|
+
next if node.equal?(root)
|
|
153
|
+
|
|
154
|
+
identifying_title_class(node) && !node.visible_text.to_s.strip.empty?
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def best_heading(headings)
|
|
159
|
+
min_name = headings.map { |node| node.name.to_s }.min
|
|
160
|
+
headings.select { |node| node.name.to_s == min_name }
|
|
161
|
+
.max_by { |node| node.visible_text.to_s.size }
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def link_node(root)
|
|
165
|
+
return if root.link?
|
|
166
|
+
|
|
167
|
+
root.find(&:link?)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def published_node(root)
|
|
171
|
+
root.find do |node|
|
|
172
|
+
next if node.equal?(root)
|
|
173
|
+
|
|
174
|
+
node.name == :time || !node.attrs.datetime.to_s.empty?
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# First class that title_node used to accept this node, not the lexicographically smallest token.
|
|
179
|
+
def identifying_title_class(node)
|
|
180
|
+
return if node.nil? || node.heading?
|
|
181
|
+
|
|
182
|
+
node.attrs.class_names.find { |name| name.match?(TITLE_CLASS_PATTERN) }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def relative_css(root, node, class_token:)
|
|
186
|
+
return if node.nil? || node.equal?(root)
|
|
187
|
+
return "#{node.name}.#{css_escape_ident(class_token)}" if class_token
|
|
188
|
+
|
|
189
|
+
relative_path_css(root, node)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# CSSOM "serialize an identifier" (CSS.escape). Class tokens are untrusted HTML.
|
|
193
|
+
def css_escape_ident(token)
|
|
194
|
+
return '\\-' if token == '-'
|
|
195
|
+
|
|
196
|
+
token.each_char.with_index.map { |char, index| css_escaped_char(char, index, token) }.join
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def css_escaped_char(char, index, token)
|
|
200
|
+
code = char.ord
|
|
201
|
+
return "\uFFFD" if code.zero?
|
|
202
|
+
return "\\#{code.to_s(16)} " if css_hex_escape?(code, index, token)
|
|
203
|
+
return char if css_ident_literal?(code)
|
|
204
|
+
|
|
205
|
+
"\\#{char}"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def css_hex_escape?(code, index, token)
|
|
209
|
+
code.between?(1, 0x1F) || code == 0x7F ||
|
|
210
|
+
(index.zero? && code.between?(0x30, 0x39)) ||
|
|
211
|
+
(index == 1 && code.between?(0x30, 0x39) && token.start_with?('-'))
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def css_ident_literal?(code)
|
|
215
|
+
code >= 0x80 || code == 0x2D || code == 0x5F ||
|
|
216
|
+
code.between?(0x30, 0x39) || code.between?(0x41, 0x5A) || code.between?(0x61, 0x7A)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def relative_path_css(root, node)
|
|
220
|
+
root_parts = path_parts(root)
|
|
221
|
+
node_parts = path_parts(node)
|
|
222
|
+
return unless node_parts[0, root_parts.length] == root_parts
|
|
223
|
+
|
|
224
|
+
rel = node_parts[root_parts.length..]
|
|
225
|
+
return if rel.nil? || rel.empty?
|
|
226
|
+
|
|
227
|
+
rel.join(' > ')
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def path_parts(node)
|
|
231
|
+
node.tag_path.to_s.split('/').reject(&:empty?)
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Selectors
|
|
2
|
+
|
|
3
|
+
How html2rss extracts feed item fields from a page when a config declares CSS `selectors:`.
|
|
4
|
+
|
|
5
|
+
## What and when
|
|
6
|
+
|
|
7
|
+
`Html2rss::Selectors` is the **orchestrator** for the traditional selector path: match item nodes, dispatch each configured field through extractors and optional post-processors, then build `Article` hashes. Prefer this when you know the list markup. Prefer `auto_source:` when you want article guessing without hand-written selectors (see {file:lib/html2rss/auto_source/README.md}).
|
|
8
|
+
|
|
9
|
+
YAML / MCP wire vocabulary is unchanged (`selectors`, `extractor`, `post_process`). Ruby type names below are the internal API.
|
|
10
|
+
|
|
11
|
+
## Live flow
|
|
12
|
+
|
|
13
|
+
1. **Parse** — HTML stays on `Response#parsed_body`. JSON responses are converted once via `JsonXml` into an HTML5 fragment so CSS selectors still run.
|
|
14
|
+
2. **Items** — `items.selector` finds each card/node. Omitted, blank, and `Selectors::DEFAULT_ITEMS_SELECTOR` expand to `Html::Navigator::MAIN_ANCHOR_SELECTOR` (permalink anchors). Any other selector, including `a` and `.card`, is used as written. `Config.prepare_defaults` fills that default, with `enhance: true`, only when `selectors:` is present. `enhance: true` fills missing fields via `Html::ArticleExtractor` (list-card enrichment). Enhancement is skipped when title, url, description, published_at, enclosures, and categories are already populated.
|
|
15
|
+
3. **Field dispatch** (private on `Selectors`) — for each selectable key (`title`, `url`, `enclosure`, `categories`, …):
|
|
16
|
+
- Regular fields → `Extractors.call` → optional `PostProcessors.call` chain
|
|
17
|
+
- Special keys → enclosure wrap, guid fan-out, multi-node categories
|
|
18
|
+
4. **Article** — values land on `Html2rss::Article` (`enclosure` YAML → `enclosures` on the Article).
|
|
19
|
+
|
|
20
|
+
```text
|
|
21
|
+
Selectors (orchestrator)
|
|
22
|
+
└─ ItemEnv
|
|
23
|
+
└─ field dispatch (private)
|
|
24
|
+
├─ Extractors::*#call(**config, xml:) + OPTIONS: [OptionSpec]
|
|
25
|
+
└─ PostProcessors::*#call + StepEnv(base_url:, time_zone:, step_config:, item_env:)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## The Three Layers of #call
|
|
29
|
+
|
|
30
|
+
| Layer | Entrypoint | Role | Inputs / Outputs |
|
|
31
|
+
| --- | --- | --- | --- |
|
|
32
|
+
| **Registry dispatcher** | `Extractors.call(config, xml)` | Resolves strategy from `NAME_TO_CLASS`, instantiates with `**config`, executes `#call` | `(config Hash, xml Node)` → extracted value |
|
|
33
|
+
| **Registry dispatcher** | `PostProcessors.call(name, value, context)` | Resolves strategy from `NAME_TO_CLASS`, passes `StepEnv`, executes `#call` | `(name, value, StepEnv)` → transformed value |
|
|
34
|
+
| **Strategy execution** | `Extractors::*#call` | Polymorphic instance execution on extractor strategies | Keyword config on `initialize`, returns extracted Object |
|
|
35
|
+
| **Strategy execution** | `PostProcessors::*#call` | Polymorphic instance execution on post-processor strategies | Reads `@value` and `@context`, returns transformed Object |
|
|
36
|
+
| **Data converter** | `JsonXml.call(object)` | Internal helper converting JSON object to XML string fragment | `(Hash / Array)` → `String` XML fragment |
|
|
37
|
+
| **Standalone sanitizer** | `SanitizeHtml.call(html, url)` | Cached helper for description/standalone sanitization | `(html, url)` → sanitized HTML String |
|
|
38
|
+
|
|
39
|
+
## Extractor vs PostProcessor
|
|
40
|
+
|
|
41
|
+
| Role | Registry | Verb | Config |
|
|
42
|
+
| --- | --- | --- | --- |
|
|
43
|
+
| Extractor | `Extractors::NAME_TO_CLASS` | `#call` / `Extractors.call` | YAML `extractor:` + strategy `OPTIONS` |
|
|
44
|
+
| Post-processor | `PostProcessors::NAME_TO_CLASS` | `#call` / `PostProcessors.call` | YAML `post_process:` steps with `name:` + `OPTIONS` |
|
|
45
|
+
|
|
46
|
+
`Extractors::Attribute` is the HTML-attribute strategy — not the field-dispatch layer (that is private on `Selectors`).
|
|
47
|
+
|
|
48
|
+
## ItemEnv and StepEnv
|
|
49
|
+
|
|
50
|
+
- **`ItemEnv`** — per-item extraction environment: node, `base_url`, scraper, `time_zone`. Nested selects reuse one env via `#select`.
|
|
51
|
+
- **`StepEnv`** — post-processor invocation bag: `step_config` (Hash), `base_url`, `time_zone`, `item_env`. Built by `ItemEnv#context_for(step_config:)`. Distinct from `OptionSpec` (static strategy introspection). Channel URL on Test / MCP is a different field.
|
|
52
|
+
|
|
53
|
+
## OptionSpec vs SchemaExport
|
|
54
|
+
|
|
55
|
+
- **`OptionSpec`** — introspection & validation SoT (`for(klass)`, `expectation_for`, `valid_type?`, `error_message`, `json_type`) with **Ruby-typed** `type:` / `required:`. Strategies own `OPTIONS: [OptionSpec, …]`.
|
|
56
|
+
- **`SchemaExport`** — JSON Schema adapter only (`for_extractor`, `for_post_processor`), consuming `OptionSpec#json_type`. `Config::IssueMapper` maps Ruby expectations → JSON via `OptionSpec.json_type_for` so ValidationIssue `expected` wire shape stays stable.
|
|
57
|
+
|
|
58
|
+
## Constraints
|
|
59
|
+
|
|
60
|
+
- Class / YAML name stays `Selectors`.
|
|
61
|
+
- Strategy verb is `#call`.
|
|
62
|
+
- `gsub` rejects a `pattern` longer than 256 characters as written, or whose star-height is greater than 1; `Html2rss.validate` and extraction share that compile.
|
|
63
|
+
|
|
64
|
+
See also {file:CONTEXT CONTEXT.md} for registry ownership and {Html2rss::Config::SelectorsValidator} for Dry validation of the selectors hash.
|
|
@@ -24,13 +24,12 @@ module Html2rss
|
|
|
24
24
|
# In case you're extracting a date or a time, consider parsing it
|
|
25
25
|
# during post processing with {PostProcessors::ParseTime}.
|
|
26
26
|
class Attribute
|
|
27
|
-
# Config-facing
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
Options = Struct.new('AttributeOptions', :selector, :attribute, keyword_init: true)
|
|
27
|
+
# Config-facing options contract (validator introspection; +selector+ is runtime-only).
|
|
28
|
+
OPTIONS = [
|
|
29
|
+
OptionSpec.new(name: :attribute, type: String)
|
|
30
|
+
].freeze
|
|
32
31
|
|
|
33
|
-
# JSON Schema description exported via +
|
|
32
|
+
# JSON Schema description exported via +schema_export+.
|
|
34
33
|
DESCRIPTION = 'Return the value of an HTML attribute on the selected element. ' \
|
|
35
34
|
'Requires sibling selector option `attribute` (attribute name).'
|
|
36
35
|
|
|
@@ -40,26 +39,25 @@ module Html2rss
|
|
|
40
39
|
].freeze
|
|
41
40
|
|
|
42
41
|
# @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
|
|
43
|
-
def self.
|
|
42
|
+
def self.schema_export = SchemaExport.for_extractor(name: :attribute, klass: self)
|
|
44
43
|
|
|
45
44
|
##
|
|
46
45
|
# Initializes the Attribute extractor.
|
|
47
46
|
#
|
|
48
47
|
# @param xml [Nokogiri::XML::Element]
|
|
49
|
-
# @param
|
|
50
|
-
# @
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
@
|
|
54
|
-
@element = Extractors.element(xml, options.selector)
|
|
48
|
+
# @param attribute [String, nil] attribute name to extract from the selected element
|
|
49
|
+
# @param selector [String, nil] CSS selector used to find the element
|
|
50
|
+
def initialize(xml, attribute: nil, selector: nil, **)
|
|
51
|
+
@attribute = attribute
|
|
52
|
+
@element = Extractors.element(xml, selector, first: true)
|
|
55
53
|
end
|
|
56
54
|
|
|
57
55
|
##
|
|
58
56
|
# Retrieves and returns the attribute's value as a string.
|
|
59
57
|
#
|
|
60
58
|
# @return [String] The value of the attribute.
|
|
61
|
-
def
|
|
62
|
-
@element
|
|
59
|
+
def call
|
|
60
|
+
@element&.[](@attribute).to_s
|
|
63
61
|
end
|
|
64
62
|
end
|
|
65
63
|
end
|
|
@@ -6,7 +6,7 @@ module Html2rss
|
|
|
6
6
|
##
|
|
7
7
|
# Returns the value of the +href+ attribute.
|
|
8
8
|
# It always returns absolute URLs. If the extracted +href+ value is a
|
|
9
|
-
# relative URL, it prepends the
|
|
9
|
+
# relative URL, it prepends the page base URL.
|
|
10
10
|
#
|
|
11
11
|
# Imagine this +a+ HTML element with a +href+ attribute:
|
|
12
12
|
#
|
|
@@ -24,12 +24,12 @@ module Html2rss
|
|
|
24
24
|
# Would return:
|
|
25
25
|
# 'http://blog-without-a-feed.example.com/posts/latest-findings'
|
|
26
26
|
class Href
|
|
27
|
-
#
|
|
28
|
-
|
|
27
|
+
# Config-facing options contract (none).
|
|
28
|
+
OPTIONS = [].freeze
|
|
29
29
|
|
|
30
|
-
# JSON Schema description exported via +
|
|
30
|
+
# JSON Schema description exported via +schema_export+.
|
|
31
31
|
DESCRIPTION = 'Return the absolute URL from the selected element\'s `href` attribute ' \
|
|
32
|
-
'(relative hrefs are resolved against the
|
|
32
|
+
'(relative hrefs are resolved against the page base URL).'
|
|
33
33
|
|
|
34
34
|
# Example extractor name values for JSON Schema +examples+.
|
|
35
35
|
EXAMPLES = [
|
|
@@ -37,29 +37,28 @@ module Html2rss
|
|
|
37
37
|
].freeze
|
|
38
38
|
|
|
39
39
|
# @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
|
|
40
|
-
def self.
|
|
40
|
+
def self.schema_export = SchemaExport.for_extractor(name: :href, klass: self)
|
|
41
41
|
|
|
42
42
|
##
|
|
43
43
|
# Initializes the Href extractor.
|
|
44
44
|
#
|
|
45
45
|
# @param xml [Nokogiri::XML::Element]
|
|
46
|
-
# @param
|
|
47
|
-
# @
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
@
|
|
51
|
-
@
|
|
52
|
-
@href = @element.attr('href').to_s
|
|
46
|
+
# @param base_url [String, Html2rss::Url, nil] page base URL for relative hrefs
|
|
47
|
+
# @param selector [String, nil] CSS selector used to find the link element
|
|
48
|
+
def initialize(xml, base_url: nil, selector: nil, **)
|
|
49
|
+
@base_url = base_url
|
|
50
|
+
@element = Extractors.element(xml, selector, first: true)
|
|
51
|
+
@href = @element&.[]('href').to_s
|
|
53
52
|
end
|
|
54
53
|
|
|
55
54
|
##
|
|
56
55
|
# Retrieves and returns the normalized absolute URL.
|
|
57
56
|
#
|
|
58
|
-
# @return [
|
|
59
|
-
def
|
|
57
|
+
# @return [Html2rss::Url, nil] The absolute URL.
|
|
58
|
+
def call
|
|
60
59
|
return nil unless @href
|
|
61
60
|
|
|
62
|
-
Url.from_relative(@href, @
|
|
61
|
+
Url.from_relative(@href, @base_url)
|
|
63
62
|
end
|
|
64
63
|
end
|
|
65
64
|
end
|
|
@@ -23,10 +23,10 @@ module Html2rss
|
|
|
23
23
|
# Always ensure to sanitize the HTML during post-processing with
|
|
24
24
|
# {PostProcessors::SanitizeHtml}.
|
|
25
25
|
class Html
|
|
26
|
-
#
|
|
27
|
-
|
|
26
|
+
# Config-facing options contract (none).
|
|
27
|
+
OPTIONS = [].freeze
|
|
28
28
|
|
|
29
|
-
# JSON Schema description exported via +
|
|
29
|
+
# JSON Schema description exported via +schema_export+.
|
|
30
30
|
DESCRIPTION = 'Return the outer HTML of the selected element. ' \
|
|
31
31
|
'Sanitize during post-processing (e.g. `sanitize_html`).'
|
|
32
32
|
|
|
@@ -36,23 +36,22 @@ module Html2rss
|
|
|
36
36
|
].freeze
|
|
37
37
|
|
|
38
38
|
# @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
|
|
39
|
-
def self.
|
|
39
|
+
def self.schema_export = SchemaExport.for_extractor(name: :html, klass: self)
|
|
40
40
|
|
|
41
41
|
##
|
|
42
42
|
# Initializes the Html extractor.
|
|
43
43
|
#
|
|
44
44
|
# @param xml [Nokogiri::XML::Element]
|
|
45
|
-
# @param
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
@element = Extractors.element(xml, options.selector)
|
|
45
|
+
# @param selector [String, nil] CSS selector used to find the element
|
|
46
|
+
def initialize(xml, selector: nil, **)
|
|
47
|
+
@element = Extractors.element(xml, selector)
|
|
49
48
|
end
|
|
50
49
|
|
|
51
50
|
##
|
|
52
51
|
# Retrieves and returns the HTML content of the element.
|
|
53
52
|
#
|
|
54
53
|
# @return [String] The HTML content.
|
|
55
|
-
def
|
|
54
|
+
def call
|
|
56
55
|
@element.to_s
|
|
57
56
|
end
|
|
58
57
|
end
|
|
@@ -16,13 +16,12 @@ module Html2rss
|
|
|
16
16
|
# Would return:
|
|
17
17
|
# 'Foobar'
|
|
18
18
|
class Static
|
|
19
|
-
# Config-facing
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
Options = Struct.new('StaticOptions', :static, keyword_init: true)
|
|
19
|
+
# Config-facing options contract (validator introspection).
|
|
20
|
+
OPTIONS = [
|
|
21
|
+
OptionSpec.new(name: :static, type: String)
|
|
22
|
+
].freeze
|
|
24
23
|
|
|
25
|
-
# JSON Schema description exported via +
|
|
24
|
+
# JSON Schema description exported via +schema_export+.
|
|
26
25
|
DESCRIPTION = 'Return a fixed value from sibling selector option `static` (no DOM read).'
|
|
27
26
|
|
|
28
27
|
# Example extractor name values for JSON Schema +examples+.
|
|
@@ -31,24 +30,23 @@ module Html2rss
|
|
|
31
30
|
].freeze
|
|
32
31
|
|
|
33
32
|
# @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
|
|
34
|
-
def self.
|
|
33
|
+
def self.schema_export = SchemaExport.for_extractor(name: :static, klass: self)
|
|
35
34
|
|
|
36
35
|
##
|
|
37
36
|
# Initializes the Static extractor.
|
|
38
37
|
#
|
|
39
38
|
# @param _xml [nil, Nokogiri::XML::Element] Unused parameter for compatibility with other extractors.
|
|
40
|
-
# @param
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
@options = options
|
|
39
|
+
# @param static [String, Symbol, nil] static value returned by this extractor
|
|
40
|
+
def initialize(_xml = nil, static: nil, **)
|
|
41
|
+
@static = static
|
|
44
42
|
end
|
|
45
43
|
|
|
46
44
|
##
|
|
47
45
|
# Retrieves and returns the static value.
|
|
48
46
|
#
|
|
49
|
-
# @return [String, Symbol] The static value
|
|
50
|
-
def
|
|
51
|
-
@
|
|
47
|
+
# @return [String, Symbol, nil] The static value.
|
|
48
|
+
def call
|
|
49
|
+
@static
|
|
52
50
|
end
|
|
53
51
|
end
|
|
54
52
|
end
|
|
@@ -21,10 +21,10 @@ module Html2rss
|
|
|
21
21
|
# Would return:
|
|
22
22
|
# 'Lorem ipsum dolor ...'
|
|
23
23
|
class Text
|
|
24
|
-
#
|
|
25
|
-
|
|
24
|
+
# Config-facing options contract (none).
|
|
25
|
+
OPTIONS = [].freeze
|
|
26
26
|
|
|
27
|
-
# JSON Schema description exported via +
|
|
27
|
+
# JSON Schema description exported via +schema_export+.
|
|
28
28
|
DESCRIPTION = 'Return collapsed visible text of the selected element (default extractor).'
|
|
29
29
|
|
|
30
30
|
# Example extractor name values for JSON Schema +examples+.
|
|
@@ -33,23 +33,22 @@ module Html2rss
|
|
|
33
33
|
].freeze
|
|
34
34
|
|
|
35
35
|
# @return [Hash{Symbol => Object}] JSON Schema fragment for this extractor name
|
|
36
|
-
def self.
|
|
36
|
+
def self.schema_export = SchemaExport.for_extractor(name: :text, klass: self)
|
|
37
37
|
|
|
38
38
|
##
|
|
39
39
|
# Initializes the Text extractor.
|
|
40
40
|
#
|
|
41
41
|
# @param xml [Nokogiri::XML::Element]
|
|
42
|
-
# @param
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
@element = Extractors.element(xml, options.selector)
|
|
42
|
+
# @param selector [String, nil] CSS selector used to find the element
|
|
43
|
+
def initialize(xml, selector: nil, **)
|
|
44
|
+
@element = Extractors.element(xml, selector)
|
|
46
45
|
end
|
|
47
46
|
|
|
48
47
|
##
|
|
49
48
|
# Retrieves and returns the text content of the element.
|
|
50
49
|
#
|
|
51
50
|
# @return [String] The text content.
|
|
52
|
-
def
|
|
51
|
+
def call
|
|
53
52
|
@element.text.to_s.strip.gsub(/\s+/, ' ')
|
|
54
53
|
end
|
|
55
54
|
end
|
|
@@ -17,35 +17,48 @@ module Html2rss
|
|
|
17
17
|
text: Text
|
|
18
18
|
}.freeze
|
|
19
19
|
|
|
20
|
-
##
|
|
21
|
-
# Maps the extractor class to its corresponding options class.
|
|
22
|
-
ITEM_OPTION_CLASSES = Hash.new do |hash, klass|
|
|
23
|
-
hash[klass] = klass.const_get(:Options)
|
|
24
|
-
end
|
|
25
|
-
|
|
26
20
|
# Extractor used when none is explicitly configured.
|
|
27
21
|
DEFAULT_EXTRACTOR = :text
|
|
28
|
-
|
|
29
22
|
class << self
|
|
30
23
|
##
|
|
31
24
|
# Retrieves an element from Nokogiri XML based on the selector.
|
|
32
25
|
#
|
|
33
|
-
#
|
|
26
|
+
# Attribute-style extractors pass +first: true+ so Nokogiri can use +at_css+
|
|
27
|
+
# when only the first match is consumed. Text/HTML keep +css+ so every
|
|
28
|
+
# match is concatenated. Category collection also uses the default.
|
|
29
|
+
#
|
|
30
|
+
# @param xml [Nokogiri::XML::Document, Nokogiri::XML::Node]
|
|
34
31
|
# @param selector [String, nil]
|
|
35
|
-
# @
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
32
|
+
# @param first [Boolean] when true, return the first match via +at_css+
|
|
33
|
+
# @return [Nokogiri::XML::Node, Nokogiri::XML::NodeSet, nil] selected XML elements
|
|
34
|
+
def element(xml, selector, first: false)
|
|
35
|
+
return xml unless selector
|
|
39
36
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
# @param xml [Nokogiri::XML::Document]
|
|
43
|
-
# @return [Object] instance of the specified item extractor class
|
|
44
|
-
def get(attribute_options, xml)
|
|
45
|
-
extractor_class = NAME_TO_CLASS[attribute_options[:extractor]&.to_sym || DEFAULT_EXTRACTOR]
|
|
46
|
-
options = ITEM_OPTION_CLASSES[extractor_class].new(attribute_options.slice(*extractor_class::Options.members))
|
|
37
|
+
first ? xml.at_css(selector) : xml.css(selector)
|
|
38
|
+
end
|
|
47
39
|
|
|
48
|
-
|
|
40
|
+
##
|
|
41
|
+
# Registry dispatcher: resolves the extractor strategy for +config+,
|
|
42
|
+
# instantiates it with options, and executes its +#call+.
|
|
43
|
+
#
|
|
44
|
+
# +base_url+ and +selector+ can be supplied without merging a new Hash
|
|
45
|
+
# into the (static) field config on every item.
|
|
46
|
+
#
|
|
47
|
+
# @param config [Hash{Symbol => Object}]
|
|
48
|
+
# Should contain at least `:extractor` (the name) and options for that extractor.
|
|
49
|
+
# @param xml [Nokogiri::XML::Node, Nokogiri::XML::NodeSet]
|
|
50
|
+
# @param base_url [String, Html2rss::Url, nil] page base URL for relative hrefs
|
|
51
|
+
# @param selector [String, nil] CSS selector override (pass +nil+ to use +xml+ as-is)
|
|
52
|
+
# @return [Object] extracted value from the strategy instance
|
|
53
|
+
def call(config, xml, base_url: nil, selector: config[:selector])
|
|
54
|
+
extractor_class = NAME_TO_CLASS[config[:extractor]&.to_sym || DEFAULT_EXTRACTOR]
|
|
55
|
+
extractor_class.new(
|
|
56
|
+
xml,
|
|
57
|
+
selector:,
|
|
58
|
+
attribute: config[:attribute],
|
|
59
|
+
static: config[:static],
|
|
60
|
+
base_url: base_url || config[:base_url]
|
|
61
|
+
).call
|
|
49
62
|
end
|
|
50
63
|
end
|
|
51
64
|
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class Selectors
|
|
5
|
+
ItemEnv = Data.define(:item, :base_url, :scraper, :time_zone)
|
|
6
|
+
|
|
7
|
+
##
|
|
8
|
+
# Per-item extraction environment: owns the article node, page +base_url+, and
|
|
9
|
+
# +time_zone+ for one extraction pass.
|
|
10
|
+
#
|
|
11
|
+
# Distinct from {StepEnv}, which is the post-processor invocation bag (+step_config+).
|
|
12
|
+
class ItemEnv
|
|
13
|
+
##
|
|
14
|
+
# Selects an attribute using this env's item and base_url.
|
|
15
|
+
#
|
|
16
|
+
# @param name [Symbol, String]
|
|
17
|
+
# @return [Object, Array<Object>]
|
|
18
|
+
def select(name) = scraper.select_in_scope(name, self)
|
|
19
|
+
|
|
20
|
+
##
|
|
21
|
+
# Builds a post-processor {StepEnv} carrying this env for nested selects.
|
|
22
|
+
#
|
|
23
|
+
# @param step_config [Hash] post-processor configuration from the selector config
|
|
24
|
+
# @option step_config [String] :name post-processor name
|
|
25
|
+
# @return [StepEnv]
|
|
26
|
+
def context_for(step_config:)
|
|
27
|
+
StepEnv.new(step_config:, item_env: self)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|