html2rss 0.30.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/html2rss/article/deduplicator.rb +2 -4
- data/lib/html2rss/article.rb +16 -39
- data/lib/html2rss/auto_source/README.md +1 -1
- data/lib/html2rss/auto_source/cleanup.rb +15 -6
- data/lib/html2rss/auto_source/scraper/html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
- data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
- data/lib/html2rss/auto_source/scraper.rb +4 -14
- data/lib/html2rss/auto_source/segment.rb +7 -4
- data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
- data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
- data/lib/html2rss/auto_source/segmenter.rb +5 -5
- data/lib/html2rss/auto_source.rb +2 -2
- data/lib/html2rss/batch.rb +3 -1
- data/lib/html2rss/capture/README.md +6 -4
- data/lib/html2rss/capture.rb +67 -35
- data/lib/html2rss/channel.rb +3 -3
- data/lib/html2rss/cli/probe_view.rb +5 -3
- data/lib/html2rss/cli/render.rb +6 -5
- data/lib/html2rss/cli/validate.rb +28 -27
- data/lib/html2rss/cli.rb +1 -1
- data/lib/html2rss/config/issue_mapper.rb +108 -0
- data/lib/html2rss/config/request_headers.rb +2 -3
- data/lib/html2rss/config/schema.rb +16 -1
- data/lib/html2rss/config/selectors_validator.rb +92 -70
- data/lib/html2rss/config/validation_issue.rb +60 -0
- data/lib/html2rss/config/validation_report.rb +55 -0
- data/lib/html2rss/config/validator.rb +10 -6
- data/lib/html2rss/config.rb +35 -113
- data/lib/html2rss/defaults.rb +42 -30
- data/lib/html2rss/doctor/botasaurus.rb +6 -4
- data/lib/html2rss/doctor.rb +8 -0
- data/lib/html2rss/error.rb +1 -1
- data/lib/html2rss/feed_builder/rss.rb +9 -14
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +6 -2
- data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
- data/lib/html2rss/feed_resolution/diag.rb +3 -1
- data/lib/html2rss/feed_resolution/options.rb +3 -1
- data/lib/html2rss/feed_resolution/policy.rb +2 -2
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +2 -4
- data/lib/html2rss/html/feed_link.rb +3 -1
- data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
- data/lib/html2rss/html/navigator.rb +10 -10
- data/lib/html2rss/html/rendering/description_builder.rb +1 -1
- data/lib/html2rss/html/sst_article_extractor.rb +8 -5
- data/lib/html2rss/link_destination/destination_facts.rb +4 -2
- data/lib/html2rss/link_destination/noise_policy.rb +10 -1
- data/lib/html2rss/link_destination/path_classifier.rb +45 -38
- data/lib/html2rss/mcp/README.md +2 -0
- data/lib/html2rss/mcp/config_argument.rb +3 -1
- data/lib/html2rss/mcp/contract.rb +6 -20
- data/lib/html2rss/mcp/outcome/playbook.rb +2 -2
- data/lib/html2rss/mcp/outcome.rb +36 -43
- data/lib/html2rss/mcp/server/tools.rb +111 -131
- data/lib/html2rss/mcp/server.rb +26 -18
- data/lib/html2rss/page_recon/README.md +5 -5
- data/lib/html2rss/page_recon/diagnostics.rb +71 -81
- data/lib/html2rss/page_recon.rb +9 -5
- data/lib/html2rss/recon.rb +19 -30
- data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
- data/lib/html2rss/request_service/botasaurus_strategy.rb +1 -1
- data/lib/html2rss/request_service/compressed_body.rb +4 -1
- data/lib/html2rss/request_service/httpx_strategy.rb +39 -28
- data/lib/html2rss/request_service/network_guard.rb +7 -5
- data/lib/html2rss/request_service/response.rb +1 -2
- data/lib/html2rss/request_service/strategy.rb +3 -8
- data/lib/html2rss/scoring/container_assessor.rb +3 -1
- data/lib/html2rss/scoring/engine.rb +8 -8
- data/lib/html2rss/scoring/observation.rb +5 -3
- data/lib/html2rss/scoring/ranked_segment.rb +3 -1
- data/lib/html2rss/scrape_target.rb +3 -1
- data/lib/html2rss/selector_candidates.rb +234 -0
- data/lib/html2rss/selectors/README.md +64 -0
- data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
- data/lib/html2rss/selectors/extractors/href.rb +15 -16
- data/lib/html2rss/selectors/extractors/html.rb +8 -9
- data/lib/html2rss/selectors/extractors/static.rb +12 -14
- data/lib/html2rss/selectors/extractors/text.rb +8 -9
- data/lib/html2rss/selectors/extractors.rb +33 -20
- data/lib/html2rss/selectors/item_env.rb +31 -0
- data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
- data/lib/html2rss/selectors/option_spec.rb +140 -0
- data/lib/html2rss/selectors/post_processors/base.rb +79 -31
- data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
- data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
- data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
- data/lib/html2rss/selectors/post_processors/template.rb +23 -19
- data/lib/html2rss/selectors/post_processors.rb +6 -5
- data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
- data/lib/html2rss/selectors/step_env.rb +24 -0
- data/lib/html2rss/selectors.rb +231 -142
- data/lib/html2rss/sst/attrs.rb +5 -3
- data/lib/html2rss/sst/document.rb +3 -1
- data/lib/html2rss/sst/index.rb +3 -1
- data/lib/html2rss/sst/node.rb +29 -10
- data/lib/html2rss/sst/normalizer.rb +12 -16
- data/lib/html2rss/sst/tags.rb +7 -7
- data/lib/html2rss/status.rb +13 -8
- data/lib/html2rss/surface_category.rb +14 -32
- data/lib/html2rss/syndication/parser.rb +2 -2
- data/lib/html2rss/test/enhance_audit.rb +1 -3
- data/lib/html2rss/test/policy.rb +106 -0
- data/lib/html2rss/test.rb +126 -173
- data/lib/html2rss/url.rb +25 -4
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +21 -13
- data/lib/tasks/config_schema.rake +2 -2
- data/schema/html2rss-config.schema.json +11 -8
- metadata +15 -6
- data/lib/html2rss/selectors/item_scope.rb +0 -30
|
@@ -4,8 +4,8 @@ module Html2rss
|
|
|
4
4
|
class AutoSource
|
|
5
5
|
class Segmenter
|
|
6
6
|
##
|
|
7
|
-
# Builds repeated-list article segments
|
|
8
|
-
#
|
|
7
|
+
# Builds repeated-list article segments by grouping cards under a shared parent.
|
|
8
|
+
# When the document has a +main+ or +[role=main]+, anchors outside it are dropped.
|
|
9
9
|
module List
|
|
10
10
|
module_function
|
|
11
11
|
|
|
@@ -30,40 +30,55 @@ module Html2rss
|
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def article_pairs(segmenter)
|
|
33
|
-
|
|
34
|
-
top_paths(segmenter, by_path).flat_map do |path|
|
|
35
|
-
by_path.fetch(path, []).filter_map do |node|
|
|
36
|
-
article_tag = parent_until_boundary(segmenter, node)
|
|
37
|
-
next unless article_tag
|
|
38
|
-
|
|
39
|
-
[article_tag, node]
|
|
40
|
-
end
|
|
41
|
-
end
|
|
33
|
+
top_groups(segmenter).flatten(1)
|
|
42
34
|
end
|
|
43
35
|
module_function :article_pairs
|
|
44
36
|
private_class_method :article_pairs
|
|
45
37
|
|
|
46
|
-
def
|
|
47
|
-
|
|
48
|
-
|
|
38
|
+
def top_groups(segmenter)
|
|
39
|
+
cards_by_parent(segmenter)
|
|
40
|
+
.select { |_parent, cards| cards.size >= segmenter.minimum_selector_frequency }
|
|
41
|
+
.max_by(segmenter.use_top_selectors) { |_parent, cards| cards.size }
|
|
42
|
+
.map(&:last)
|
|
43
|
+
end
|
|
44
|
+
module_function :top_groups
|
|
45
|
+
private_class_method :top_groups
|
|
46
|
+
|
|
47
|
+
def cards_by_parent(segmenter) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
48
|
+
index = segmenter.index
|
|
49
|
+
main = main_landmark(index)
|
|
50
|
+
groups = {}.compare_by_identity
|
|
51
|
+
|
|
52
|
+
index.each_node do |node|
|
|
49
53
|
next unless node.link?
|
|
50
|
-
next if
|
|
54
|
+
next if index.ignored_chrome?(node)
|
|
51
55
|
next unless relevant_anchor?(segmenter, node)
|
|
56
|
+
next if main && !inside_main?(index, node, main)
|
|
57
|
+
|
|
58
|
+
card = parent_until_boundary(segmenter, node)
|
|
59
|
+
next unless card
|
|
52
60
|
|
|
53
|
-
|
|
61
|
+
parent = index.parent_of(card)
|
|
62
|
+
next unless parent
|
|
63
|
+
|
|
64
|
+
(groups[parent] ||= []) << [card, node]
|
|
54
65
|
end
|
|
55
|
-
|
|
66
|
+
groups
|
|
67
|
+
end
|
|
68
|
+
module_function :cards_by_parent
|
|
69
|
+
private_class_method :cards_by_parent
|
|
70
|
+
|
|
71
|
+
def main_landmark(index)
|
|
72
|
+
index.each_node.find { |node| node.name == :main || node.attrs.raw['role'] == 'main' }
|
|
56
73
|
end
|
|
57
|
-
module_function :
|
|
58
|
-
private_class_method :
|
|
74
|
+
module_function :main_landmark
|
|
75
|
+
private_class_method :main_landmark
|
|
59
76
|
|
|
60
|
-
def
|
|
61
|
-
|
|
62
|
-
.max_by(segmenter.use_top_selectors) { |_path, nodes| nodes.size }
|
|
63
|
-
.map(&:first)
|
|
77
|
+
def inside_main?(index, node, main)
|
|
78
|
+
node.equal?(main) || index.descendant_of?(node, main)
|
|
64
79
|
end
|
|
65
|
-
module_function :
|
|
66
|
-
private_class_method :
|
|
80
|
+
module_function :inside_main?
|
|
81
|
+
private_class_method :inside_main?
|
|
67
82
|
|
|
68
83
|
def relevant_anchor?(segmenter, node)
|
|
69
84
|
facts = segmenter.link_resolver.destination_facts(node)
|
|
@@ -77,12 +92,15 @@ module Html2rss
|
|
|
77
92
|
module_function :relevant_anchor?
|
|
78
93
|
private_class_method :relevant_anchor?
|
|
79
94
|
|
|
95
|
+
BOUNDARY_TAGS = Set[:body, :html].freeze
|
|
96
|
+
private_constant :BOUNDARY_TAGS
|
|
97
|
+
|
|
80
98
|
def parent_until_boundary(segmenter, node)
|
|
81
99
|
index = segmenter.index
|
|
82
|
-
link_counts = Hash.new { |hash, curr| hash[curr] =
|
|
100
|
+
link_counts = Hash.new { |hash, curr| hash[curr] = count_relevant_links(segmenter, curr) }
|
|
83
101
|
|
|
84
102
|
index.parent_until(node, lambda { |curr|
|
|
85
|
-
return true if
|
|
103
|
+
return true if BOUNDARY_TAGS.include?(curr.name)
|
|
86
104
|
return false if index.ignored_chrome?(curr)
|
|
87
105
|
|
|
88
106
|
parent = index.parent_of(curr)
|
|
@@ -92,11 +110,16 @@ module Html2rss
|
|
|
92
110
|
module_function :parent_until_boundary
|
|
93
111
|
private_class_method :parent_until_boundary
|
|
94
112
|
|
|
95
|
-
def
|
|
96
|
-
|
|
113
|
+
def count_relevant_links(segmenter, node)
|
|
114
|
+
# Chrome inside a card must not stop the walk before the shared list parent.
|
|
115
|
+
if node.link?
|
|
116
|
+
relevant_anchor?(segmenter, node) ? 1 : 0
|
|
117
|
+
else
|
|
118
|
+
node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
|
|
119
|
+
end
|
|
97
120
|
end
|
|
98
|
-
module_function :
|
|
99
|
-
private_class_method :
|
|
121
|
+
module_function :count_relevant_links
|
|
122
|
+
private_class_method :count_relevant_links
|
|
100
123
|
end
|
|
101
124
|
end
|
|
102
125
|
end
|
|
@@ -7,7 +7,7 @@ module Html2rss
|
|
|
7
7
|
#
|
|
8
8
|
# Strategies:
|
|
9
9
|
# - +:semantic+ — leaf article/section/li/tr/div containers with a primary link
|
|
10
|
-
# - +:list+ — repeated
|
|
10
|
+
# - +:list+ — repeated cards grouped by parent (`main` is a filter when present)
|
|
11
11
|
# - +:cluster+ — class / structure clustering for anchorless card grids
|
|
12
12
|
class Segmenter
|
|
13
13
|
# @param document [SST::Document]
|
|
@@ -68,11 +68,11 @@ module Html2rss
|
|
|
68
68
|
# @param container [SST::Node]
|
|
69
69
|
# @return [Boolean]
|
|
70
70
|
def landmark_ancestor?(anchor, container)
|
|
71
|
-
cache = (@landmark_cache ||= {})
|
|
72
|
-
|
|
73
|
-
return
|
|
71
|
+
cache = (@landmark_cache ||= {}.compare_by_identity)
|
|
72
|
+
anchor_cache = (cache[anchor] ||= {}.compare_by_identity)
|
|
73
|
+
return anchor_cache[container] if anchor_cache.key?(container)
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
anchor_cache[container] = compute_landmark_ancestor?(anchor:, container:)
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
private
|
data/lib/html2rss/auto_source.rb
CHANGED
|
@@ -93,7 +93,7 @@ module Html2rss
|
|
|
93
93
|
Scraper::SCRAPERS.sum do |scraper|
|
|
94
94
|
if config.dig(:scraper, scraper.options_key, :enabled)
|
|
95
95
|
opts = config.dig(:scraper, scraper.options_key)
|
|
96
|
-
scraper.
|
|
96
|
+
scraper.request_slots(opts)
|
|
97
97
|
else
|
|
98
98
|
0
|
|
99
99
|
end
|
|
@@ -180,7 +180,7 @@ module Html2rss
|
|
|
180
180
|
captured_responses:
|
|
181
181
|
)
|
|
182
182
|
next unless instance
|
|
183
|
-
next unless
|
|
183
|
+
next unless instance.extractable?
|
|
184
184
|
|
|
185
185
|
matched = true
|
|
186
186
|
articles.concat(run_scraper(instance))
|
data/lib/html2rss/batch.rb
CHANGED
|
@@ -10,8 +10,10 @@ module Html2rss
|
|
|
10
10
|
# Default worker threads for batch operations.
|
|
11
11
|
DEFAULT_CONCURRENCY = 5
|
|
12
12
|
|
|
13
|
+
BatchResult = Data.define(:total, :successful, :results)
|
|
14
|
+
|
|
13
15
|
# Value object representing the outcome of a batch operation.
|
|
14
|
-
BatchResult
|
|
16
|
+
class BatchResult
|
|
15
17
|
##
|
|
16
18
|
# @return [Hash{Symbol => Object}]
|
|
17
19
|
def to_h
|
|
@@ -27,7 +27,9 @@ File.write('my-feed.yml', result.yaml)
|
|
|
27
27
|
rss = Html2rss.apply(result.config)
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
-
`Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`).
|
|
30
|
+
`Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`, `candidates`).
|
|
31
|
+
|
|
32
|
+
`#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
|
|
31
33
|
|
|
32
34
|
**Wire vs internal:** user-facing **`Html2rss.apply`** ships RSS from a config Hash. Pipeline internals **`Html2rss.feed` / `feed_result`** stay unchanged — use `apply` in CLI/MCP/docs examples, not `feed`.
|
|
33
35
|
|
|
@@ -58,13 +60,13 @@ html2rss capture https://example.com --explain # quality JSON on stderr; YAML
|
|
|
58
60
|
1. **Request** — `FeedPipeline` (AutoFallback when `:auto`)
|
|
59
61
|
2. **Discover** — AutoSource extracts admitted articles
|
|
60
62
|
3. **Segment** — try SST Segmenter strategies `:list` → `:cluster` → `:semantic`
|
|
61
|
-
4. **Gate** —
|
|
62
|
-
5. **Assemble** — `{ items: { selector:, enhance:
|
|
63
|
+
4. **Gate** — keep a derived items selector when ≥ `MIN_SELECTOR_MATCHES` (2) articles match
|
|
64
|
+
5. **Assemble** — `{ items: { selector:, enhance: } }` plus channel. HTML derivation misses fall back to `Selectors::DEFAULT_ITEMS_SELECTOR` (`segment_strategy: :default`) instead of omitting selectors.
|
|
63
65
|
|
|
64
66
|
## Constraints
|
|
65
67
|
|
|
66
68
|
- Selector quality depends on page structure; treat output as a first draft.
|
|
67
|
-
- When
|
|
69
|
+
- When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs, or a rescued `ArgumentError`), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
|
|
68
70
|
- Botasaurus hops need `BOTASAURUS_SCRAPER_URL`.
|
|
69
71
|
|
|
70
72
|
See also {Html2rss::AutoSource} for article discovery and {Html2rss::FeedPipeline} for the `:auto` request chain.
|
data/lib/html2rss/capture.rb
CHANGED
|
@@ -25,12 +25,15 @@ module Html2rss
|
|
|
25
25
|
CHROME_DROP_THRESHOLD = 3
|
|
26
26
|
private_constant :CHROME_DROP_THRESHOLD
|
|
27
27
|
|
|
28
|
-
##
|
|
29
|
-
# Result of a capture operation (config plus quality meta).
|
|
30
28
|
CaptureResult = Data.define(
|
|
31
29
|
:config, :yaml, :articles_count, :channel_title, :has_selectors, :segment_strategy,
|
|
32
|
-
:admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url
|
|
33
|
-
|
|
30
|
+
:admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url,
|
|
31
|
+
:candidates
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
##
|
|
35
|
+
# Result of a capture operation (config plus quality meta).
|
|
36
|
+
class CaptureResult
|
|
34
37
|
# rubocop:disable Metrics/ParameterLists
|
|
35
38
|
##
|
|
36
39
|
# @param config [Hash]
|
|
@@ -44,9 +47,10 @@ module Html2rss
|
|
|
44
47
|
# @param inferred_topics [Array<String>]
|
|
45
48
|
# @param native_feed [String, nil]
|
|
46
49
|
# @param suggested_channel_url [String, nil]
|
|
50
|
+
# @param candidates [Hash{Symbol=>Array}, nil] ranked selector buckets from {SelectorCandidates}
|
|
47
51
|
def initialize(config:, articles_count:, channel_title:, has_selectors:, segment_strategy:, # rubocop:disable Metrics/MethodLength
|
|
48
52
|
yaml: nil, admission_drops: {}, selected_strategy: nil, inferred_topics: [], native_feed: nil,
|
|
49
|
-
suggested_channel_url: nil)
|
|
53
|
+
suggested_channel_url: nil, candidates: nil)
|
|
50
54
|
super(
|
|
51
55
|
config:,
|
|
52
56
|
yaml: yaml || "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
|
|
@@ -58,7 +62,8 @@ module Html2rss
|
|
|
58
62
|
selected_strategy:,
|
|
59
63
|
inferred_topics:,
|
|
60
64
|
native_feed:,
|
|
61
|
-
suggested_channel_url
|
|
65
|
+
suggested_channel_url:,
|
|
66
|
+
candidates: candidates || SelectorCandidates.empty
|
|
62
67
|
)
|
|
63
68
|
end
|
|
64
69
|
# rubocop:enable Metrics/ParameterLists
|
|
@@ -127,7 +132,7 @@ module Html2rss
|
|
|
127
132
|
def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
128
133
|
outcome = FeedPipeline.new(raw_config).to_outcome
|
|
129
134
|
@admission_drops = outcome.admission_drops
|
|
130
|
-
selectors, segment_strategy = derive_selectors(outcome.response, outcome.articles)
|
|
135
|
+
selectors, segment_strategy, candidates = derive_selectors(outcome.response, outcome.articles)
|
|
131
136
|
ch_title = @title || channel_title_from(outcome.response)
|
|
132
137
|
topics = @topics || infer_topics("#{@url} #{ch_title}")
|
|
133
138
|
native_feed = probe_native_feed(outcome.response) unless @force
|
|
@@ -151,7 +156,8 @@ module Html2rss
|
|
|
151
156
|
selected_strategy: outcome.selected_strategy,
|
|
152
157
|
inferred_topics: topics,
|
|
153
158
|
native_feed:,
|
|
154
|
-
suggested_channel_url: suggested_channel_url(outcome)
|
|
159
|
+
suggested_channel_url: suggested_channel_url(outcome),
|
|
160
|
+
candidates:
|
|
155
161
|
)
|
|
156
162
|
end
|
|
157
163
|
|
|
@@ -254,40 +260,68 @@ module Html2rss
|
|
|
254
260
|
{ strategy: :local_file, request: { local_file_path: @local_file_path } }
|
|
255
261
|
end
|
|
256
262
|
|
|
257
|
-
# @return [Array(Hash, Symbol, nil)] selectors
|
|
263
|
+
# @return [Array(Hash, Symbol, nil, Hash)] selectors, segment strategy, candidate buckets
|
|
258
264
|
def derive_selectors(response, articles)
|
|
259
|
-
return hint_selectors if @items_selector_hint
|
|
260
|
-
return [{}, nil
|
|
265
|
+
return [*hint_selectors, SelectorCandidates.empty] if @items_selector_hint
|
|
266
|
+
return [{}, nil, SelectorCandidates.empty] unless response.html_response?
|
|
267
|
+
return [*default_selectors, SelectorCandidates.empty] if articles.empty?
|
|
261
268
|
|
|
262
269
|
sst = SST::Normalizer.call(response.body)
|
|
263
|
-
return [
|
|
270
|
+
return [*default_selectors, SelectorCandidates.empty] unless sst
|
|
264
271
|
|
|
265
272
|
select_enhance_selectors(sst, articles)
|
|
266
273
|
rescue ArgumentError => error
|
|
267
274
|
Log.warn("Capture selector derivation failed: #{error.message}")
|
|
268
|
-
[
|
|
275
|
+
[*default_selectors, SelectorCandidates.empty]
|
|
269
276
|
end
|
|
270
277
|
|
|
271
278
|
def hint_selectors
|
|
272
279
|
[{ items: { selector: @items_selector_hint, enhance: resolve_enhance } }, :hint]
|
|
273
280
|
end
|
|
274
281
|
|
|
275
|
-
def
|
|
282
|
+
def default_selectors
|
|
283
|
+
[{ items: { selector: Selectors::DEFAULT_ITEMS_SELECTOR, enhance: resolve_enhance } }, :default]
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def select_enhance_selectors(sst, articles)
|
|
287
|
+
evidence = collect_items_evidence(sst, articles)
|
|
288
|
+
candidates = SelectorCandidates.call(items_evidence: evidence, min_matches: MIN_SELECTOR_MATCHES)
|
|
289
|
+
top = candidates[:items].first
|
|
290
|
+
return [*default_selectors, candidates] unless top
|
|
291
|
+
|
|
292
|
+
strategy = evidence.find { |entry| entry[:selector] == top[:selector] }&.fetch(:strategy)
|
|
293
|
+
items = { selector: top[:selector], enhance: top[:enhance] }
|
|
294
|
+
[{ items: }, strategy, candidates]
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def collect_items_evidence(sst, articles)
|
|
276
298
|
link_resolver = Scoring::LinkResolver.new(@url)
|
|
277
299
|
enhance = resolve_enhance
|
|
278
300
|
|
|
279
|
-
SEGMENT_STRATEGIES.
|
|
301
|
+
SEGMENT_STRATEGIES.flat_map do |strategy|
|
|
280
302
|
segments = AutoSource::Segmenter.call(
|
|
281
303
|
sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
|
|
282
304
|
)
|
|
283
305
|
matched = match_segments_to_articles(segments, articles)
|
|
284
|
-
|
|
285
|
-
|
|
306
|
+
next [] if matched.size < MIN_SELECTOR_MATCHES
|
|
307
|
+
|
|
308
|
+
items_evidence_for(matched, strategy:, enhance:)
|
|
309
|
+
end
|
|
310
|
+
end
|
|
286
311
|
|
|
287
|
-
|
|
312
|
+
def items_evidence_for(matched, strategy:, enhance:)
|
|
313
|
+
roots = lift_heading_link_roots(matched.map { |pair| pair[:segment].root_node })
|
|
314
|
+
selector_variants_for(roots).map do |kind, selector|
|
|
315
|
+
{ selector:, enhance:, strategy:, kind:, match_count: matched.size, roots: }
|
|
288
316
|
end
|
|
317
|
+
end
|
|
289
318
|
|
|
290
|
-
|
|
319
|
+
def selector_variants_for(roots)
|
|
320
|
+
[
|
|
321
|
+
[:shared_class, shared_class_items_selector(roots)],
|
|
322
|
+
[:unique_tag, unique_tag_items_selector(roots)],
|
|
323
|
+
[:path, path_items_selector(roots)]
|
|
324
|
+
].filter_map { |kind, selector| [kind, selector] if selector }
|
|
291
325
|
end
|
|
292
326
|
|
|
293
327
|
def match_segments_to_articles(segments, articles)
|
|
@@ -322,21 +356,6 @@ module Html2rss
|
|
|
322
356
|
segment.root_node.visible_text.to_s.strip
|
|
323
357
|
end
|
|
324
358
|
|
|
325
|
-
def items_selector(matched)
|
|
326
|
-
return nil if matched.empty?
|
|
327
|
-
|
|
328
|
-
roots = lift_heading_link_roots(matched.map { |m| m[:segment].root_node })
|
|
329
|
-
shared = shared_class_items_selector(roots)
|
|
330
|
-
return shared if shared
|
|
331
|
-
|
|
332
|
-
paths = roots.map(&:tag_path)
|
|
333
|
-
common = common_path_prefix(paths)
|
|
334
|
-
tag_path = common.empty? ? paths.first.to_s : common
|
|
335
|
-
return nil if tag_path.empty?
|
|
336
|
-
|
|
337
|
-
css_from_trimmed_tag_path(tag_path)
|
|
338
|
-
end
|
|
339
|
-
|
|
340
359
|
def lift_heading_link_roots(roots)
|
|
341
360
|
roots.map { |root| lift_heading_link_root(root, roots) }
|
|
342
361
|
end
|
|
@@ -371,7 +390,6 @@ module Html2rss
|
|
|
371
390
|
|
|
372
391
|
def wrapping_anchor_root?(node)
|
|
373
392
|
return false unless node.name.to_s == 'a'
|
|
374
|
-
return false unless node.respond_to?(:find)
|
|
375
393
|
|
|
376
394
|
tags = Html::Navigator::WRAPPING_ANCHOR_CHILD_TAGS
|
|
377
395
|
node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
|
|
@@ -396,6 +414,20 @@ module Html2rss
|
|
|
396
414
|
"#{roots.first.name}.#{shared.min}"
|
|
397
415
|
end
|
|
398
416
|
|
|
417
|
+
def unique_tag_items_selector(roots)
|
|
418
|
+
names = roots.map { |root| root.name.to_s }.uniq
|
|
419
|
+
names.join(', ') if names.size > 1
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
def path_items_selector(roots)
|
|
423
|
+
paths = roots.map(&:tag_path)
|
|
424
|
+
common = common_path_prefix(paths)
|
|
425
|
+
tag_path = common.empty? ? paths.first.to_s : common
|
|
426
|
+
return if tag_path.empty?
|
|
427
|
+
|
|
428
|
+
css_from_trimmed_tag_path(tag_path)
|
|
429
|
+
end
|
|
430
|
+
|
|
399
431
|
def css_from_trimmed_tag_path(tag_path)
|
|
400
432
|
segments = tag_path.split('/').reject(&:empty?)
|
|
401
433
|
trimmed = trim_leading_items_segments(segments)
|
data/lib/html2rss/channel.rb
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require 'time'
|
|
4
4
|
|
|
5
|
-
# rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
|
|
6
5
|
module Html2rss
|
|
6
|
+
Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author)
|
|
7
|
+
|
|
7
8
|
##
|
|
8
9
|
# Materialized channel metadata for feed builders and {FeedResult}.
|
|
9
10
|
#
|
|
@@ -27,7 +28,7 @@ module Html2rss
|
|
|
27
28
|
# @return [Html2rss::Url, nil]
|
|
28
29
|
# @!attribute [r] author
|
|
29
30
|
# @return [String, nil]
|
|
30
|
-
Channel
|
|
31
|
+
class Channel # rubocop:disable Metrics/ClassLength -- Channel value object owns extract + freeze + Marshal
|
|
31
32
|
class << self
|
|
32
33
|
##
|
|
33
34
|
# Materializes channel attributes from an HTTP response and optional overrides.
|
|
@@ -193,5 +194,4 @@ module Html2rss
|
|
|
193
194
|
Time.httpdate(value)
|
|
194
195
|
end
|
|
195
196
|
end
|
|
196
|
-
# rubocop:enable Metrics/ModuleLength
|
|
197
197
|
end
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
class CLI
|
|
5
|
-
##
|
|
6
|
-
# Display-only probe facts for inspect and recon text cards.
|
|
7
5
|
ProbeView = Data.define(
|
|
8
6
|
:requested,
|
|
9
7
|
:final,
|
|
@@ -15,7 +13,11 @@ module Html2rss
|
|
|
15
13
|
:alternate_feeds,
|
|
16
14
|
:notes,
|
|
17
15
|
:strategy
|
|
18
|
-
)
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
##
|
|
19
|
+
# Display-only probe facts for inspect and recon text cards.
|
|
20
|
+
class ProbeView
|
|
19
21
|
class << self
|
|
20
22
|
##
|
|
21
23
|
# @param data [Hash] inspect wire payload
|
data/lib/html2rss/cli/render.rb
CHANGED
|
@@ -93,9 +93,11 @@ module Html2rss
|
|
|
93
93
|
end
|
|
94
94
|
end
|
|
95
95
|
else
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
result.
|
|
96
|
+
$stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
|
|
97
|
+
$stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
|
|
98
|
+
result.validation_issues&.each do |issue|
|
|
99
|
+
$stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
|
|
100
|
+
end
|
|
99
101
|
end
|
|
100
102
|
end
|
|
101
103
|
|
|
@@ -115,8 +117,7 @@ module Html2rss
|
|
|
115
117
|
if view.final && view.final != view.requested
|
|
116
118
|
puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
|
|
117
119
|
end
|
|
118
|
-
|
|
119
|
-
puts " Surface: #{surface} (#{view.articles_count} articles)"
|
|
120
|
+
puts " Surface: #{view.surface} (#{view.articles_count} articles)"
|
|
120
121
|
end
|
|
121
122
|
private_class_method :probe_lines
|
|
122
123
|
end
|
|
@@ -29,11 +29,10 @@ module Html2rss
|
|
|
29
29
|
if result.success?
|
|
30
30
|
puts(target_files.size == 1 ? 'Configuration is valid' : "ok #{file}") unless quiet
|
|
31
31
|
else
|
|
32
|
-
|
|
33
|
-
raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
|
|
32
|
+
raise Thor::Error, format_failure(result) if target_files.size == 1
|
|
34
33
|
|
|
35
34
|
warn "FAIL #{file}"
|
|
36
|
-
|
|
35
|
+
result.issues.each { |issue| warn " #{issue}" }
|
|
37
36
|
failed << file
|
|
38
37
|
end
|
|
39
38
|
end
|
|
@@ -57,35 +56,37 @@ module Html2rss
|
|
|
57
56
|
resolved.uniq
|
|
58
57
|
end
|
|
59
58
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
|
|
59
|
+
class << self
|
|
60
|
+
private
|
|
63
61
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
62
|
+
def run_named_feed(file, feed_name, params:, quiet:)
|
|
63
|
+
result = Html2rss.validate(file, feed_name, params:)
|
|
64
|
+
raise Thor::Error, format_failure(result) unless result.success?
|
|
65
|
+
|
|
66
|
+
puts 'Configuration is valid' unless quiet
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def validate_file(file, params:)
|
|
70
|
+
if file == '-'
|
|
71
|
+
Html2rss.validate($stdin.read, params:)
|
|
72
|
+
else
|
|
73
|
+
Html2rss.validate(file, params:)
|
|
74
|
+
end
|
|
74
75
|
end
|
|
75
|
-
end
|
|
76
|
-
module_function :validate_file
|
|
77
|
-
private_class_method :validate_file
|
|
78
76
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
return true if path.end_with?('.yml', '.yaml')
|
|
83
|
-
return true if path.include?('/') || path.include?('*')
|
|
77
|
+
def format_failure(report)
|
|
78
|
+
"Invalid configuration: #{report}"
|
|
79
|
+
end
|
|
84
80
|
|
|
85
|
-
|
|
81
|
+
def path_like_config?(arg)
|
|
82
|
+
path = arg.to_s
|
|
83
|
+
return true if path.start_with?('-')
|
|
84
|
+
return true if path.end_with?('.yml', '.yaml')
|
|
85
|
+
return true if path.include?('/') || path.include?('*')
|
|
86
|
+
|
|
87
|
+
false
|
|
88
|
+
end
|
|
86
89
|
end
|
|
87
|
-
module_function :path_like_config?
|
|
88
|
-
private_class_method :path_like_config?
|
|
89
90
|
end
|
|
90
91
|
end
|
|
91
92
|
end
|
data/lib/html2rss/cli.rb
CHANGED
|
@@ -300,7 +300,7 @@ module Html2rss
|
|
|
300
300
|
def filter_recon_results(results, verdict_filter)
|
|
301
301
|
return results unless verdict_filter
|
|
302
302
|
|
|
303
|
-
expected = Recon::Verdict
|
|
303
|
+
expected = Recon::Verdict[verdict_filter.downcase]
|
|
304
304
|
results.select { |r| r.verdict == expected }
|
|
305
305
|
rescue ArgumentError => error
|
|
306
306
|
raise Thor::Error, error.message
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class Config
|
|
5
|
+
##
|
|
6
|
+
# Maps internal Dry::Validation messages to {ValidationIssue} (predicate → code).
|
|
7
|
+
# Not a public API — Dry stays inside Validator / this mapper.
|
|
8
|
+
module IssueMapper
|
|
9
|
+
# Dry schema predicates → closed {ValidationIssue::CODES}.
|
|
10
|
+
PREDICATE_CODES = {
|
|
11
|
+
key?: :missing_key,
|
|
12
|
+
filled?: :missing_key,
|
|
13
|
+
str?: :type_mismatch,
|
|
14
|
+
int?: :type_mismatch,
|
|
15
|
+
bool?: :type_mismatch,
|
|
16
|
+
hash?: :type_mismatch,
|
|
17
|
+
array?: :type_mismatch,
|
|
18
|
+
type?: :type_mismatch,
|
|
19
|
+
format?: :invalid_value,
|
|
20
|
+
included_in?: :unknown_value,
|
|
21
|
+
gt?: :constraint,
|
|
22
|
+
gteq?: :constraint,
|
|
23
|
+
lt?: :constraint,
|
|
24
|
+
lteq?: :constraint,
|
|
25
|
+
max_size?: :constraint,
|
|
26
|
+
min_size?: :constraint
|
|
27
|
+
}.freeze
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
##
|
|
31
|
+
# @param dry_result [Dry::Validation::Result]
|
|
32
|
+
# @param values [Hash, nil] config under validation (for actual/expected dig)
|
|
33
|
+
# @return [Html2rss::Config::ValidationReport]
|
|
34
|
+
def from(dry_result, values: nil)
|
|
35
|
+
return ValidationReport.ok if dry_result.success?
|
|
36
|
+
|
|
37
|
+
ValidationReport.failure(dry_result.errors.map { |message| issue_for(message, values:) })
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
##
|
|
41
|
+
# @param message [String]
|
|
42
|
+
# @return [Html2rss::Config::ValidationReport]
|
|
43
|
+
def parse_failure(message)
|
|
44
|
+
ValidationReport.failure(
|
|
45
|
+
[
|
|
46
|
+
ValidationIssue.new(path: %i[parse], code: :parse, message:)
|
|
47
|
+
]
|
|
48
|
+
)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def issue_for(message, values:)
|
|
54
|
+
path = Array(message.path).compact
|
|
55
|
+
ValidationIssue.new(
|
|
56
|
+
path:,
|
|
57
|
+
code: code_for(message.predicate),
|
|
58
|
+
message: message.text,
|
|
59
|
+
expected: expected_for(path, values),
|
|
60
|
+
actual: actual_for(path, values, message)
|
|
61
|
+
)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def code_for(predicate)
|
|
65
|
+
return :invalid_value if predicate.nil?
|
|
66
|
+
|
|
67
|
+
PREDICATE_CODES.fetch(predicate) do
|
|
68
|
+
raise ArgumentError, "unmapped Dry validation predicate: #{predicate.inspect}"
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def actual_for(path, values, message)
|
|
73
|
+
dug = dig_path(values, path)
|
|
74
|
+
return dug unless dug.nil?
|
|
75
|
+
return message.input if message.is_a?(Dry::Schema::Message) && !message.input.nil?
|
|
76
|
+
|
|
77
|
+
nil
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def expected_for(path, values)
|
|
81
|
+
return if values.nil? || path.empty?
|
|
82
|
+
return unless path.last.is_a?(Symbol)
|
|
83
|
+
|
|
84
|
+
parent = dig_path(values, path[0...-1])
|
|
85
|
+
return unless parent.is_a?(Hash)
|
|
86
|
+
|
|
87
|
+
option = Selectors::OptionSpec.option_for(parent, path.last)
|
|
88
|
+
return unless option
|
|
89
|
+
|
|
90
|
+
{
|
|
91
|
+
type: option.json_type,
|
|
92
|
+
required: option.required
|
|
93
|
+
}
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def dig_path(values, path)
|
|
97
|
+
return if values.nil? || path.empty?
|
|
98
|
+
|
|
99
|
+
path.reduce(values) do |cursor, segment|
|
|
100
|
+
break unless cursor.is_a?(Hash) || (cursor.is_a?(Array) && segment.is_a?(Integer))
|
|
101
|
+
|
|
102
|
+
cursor[segment]
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|