html2rss 0.29.1 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -8
- data/html2rss.gemspec +1 -3
- data/lib/html2rss/article/deduplicator.rb +2 -4
- data/lib/html2rss/article.rb +16 -39
- data/lib/html2rss/auto_source/README.md +1 -1
- data/lib/html2rss/auto_source/cleanup.rb +15 -6
- data/lib/html2rss/auto_source/scraper/html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +5 -0
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +8 -1
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -0
- data/lib/html2rss/auto_source/scraper/microformats2.rb +8 -0
- data/lib/html2rss/auto_source/scraper/native_feed.rb +3 -0
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +4 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +8 -0
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +5 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -0
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -0
- data/lib/html2rss/auto_source/scraper.rb +4 -14
- data/lib/html2rss/auto_source/segment.rb +7 -4
- data/lib/html2rss/auto_source/segmenter/cluster.rb +22 -23
- data/lib/html2rss/auto_source/segmenter/list.rb +54 -31
- data/lib/html2rss/auto_source/segmenter.rb +5 -5
- data/lib/html2rss/auto_source.rb +2 -2
- data/lib/html2rss/batch.rb +6 -4
- data/lib/html2rss/capture/README.md +7 -5
- data/lib/html2rss/capture.rb +68 -36
- data/lib/html2rss/channel.rb +3 -3
- data/lib/html2rss/cli/probe_view.rb +5 -3
- data/lib/html2rss/cli/render.rb +6 -5
- data/lib/html2rss/cli/validate.rb +28 -27
- data/lib/html2rss/cli.rb +1 -1
- data/lib/html2rss/config/issue_mapper.rb +108 -0
- data/lib/html2rss/config/request_headers.rb +6 -4
- data/lib/html2rss/config/schema.rb +16 -1
- data/lib/html2rss/config/selectors_validator.rb +92 -70
- data/lib/html2rss/config/validation_issue.rb +60 -0
- data/lib/html2rss/config/validation_report.rb +55 -0
- data/lib/html2rss/config/validator.rb +10 -6
- data/lib/html2rss/config.rb +35 -101
- data/lib/html2rss/defaults.rb +42 -30
- data/lib/html2rss/doctor/botasaurus.rb +19 -9
- data/lib/html2rss/doctor.rb +8 -0
- data/lib/html2rss/error.rb +1 -1
- data/lib/html2rss/feed_builder/rss.rb +9 -14
- data/lib/html2rss/feed_pipeline/README.md +8 -8
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +1 -1
- data/lib/html2rss/feed_pipeline/runtime_policy.rb +14 -10
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +7 -3
- data/lib/html2rss/feed_resolution/candidate_generator.rb +1 -4
- data/lib/html2rss/feed_resolution/diag.rb +3 -1
- data/lib/html2rss/feed_resolution/options.rb +3 -1
- data/lib/html2rss/feed_resolution/policy.rb +2 -2
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +2 -4
- data/lib/html2rss/html/feed_link.rb +3 -1
- data/lib/html2rss/html/navigator/text_extractor.rb +1 -1
- data/lib/html2rss/html/navigator.rb +10 -10
- data/lib/html2rss/html/rendering/description_builder.rb +1 -1
- data/lib/html2rss/html/sst_article_extractor.rb +8 -5
- data/lib/html2rss/link_destination/destination_facts.rb +4 -2
- data/lib/html2rss/link_destination/noise_policy.rb +10 -1
- data/lib/html2rss/link_destination/path_classifier.rb +45 -38
- data/lib/html2rss/mcp/README.md +5 -3
- data/lib/html2rss/mcp/config_argument.rb +3 -1
- data/lib/html2rss/mcp/contract.rb +15 -25
- data/lib/html2rss/mcp/outcome/playbook.rb +7 -7
- data/lib/html2rss/mcp/outcome.rb +36 -43
- data/lib/html2rss/mcp/server/tools.rb +112 -132
- data/lib/html2rss/mcp/server.rb +27 -19
- data/lib/html2rss/page_recon/README.md +5 -5
- data/lib/html2rss/page_recon/diagnostics.rb +71 -81
- data/lib/html2rss/page_recon.rb +9 -5
- data/lib/html2rss/recon.rb +20 -31
- data/lib/html2rss/request_service/botasaurus_contract.rb +11 -12
- data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
- data/lib/html2rss/request_service/compressed_body.rb +17 -9
- data/lib/html2rss/request_service/httpx_strategy.rb +239 -0
- data/lib/html2rss/request_service/network_guard.rb +7 -5
- data/lib/html2rss/request_service/response.rb +1 -2
- data/lib/html2rss/request_service/response_guard.rb +0 -16
- data/lib/html2rss/request_service/strategy.rb +22 -11
- data/lib/html2rss/request_service.rb +46 -85
- data/lib/html2rss/scoring/container_assessor.rb +3 -1
- data/lib/html2rss/scoring/engine.rb +8 -8
- data/lib/html2rss/scoring/observation.rb +5 -3
- data/lib/html2rss/scoring/ranked_segment.rb +3 -1
- data/lib/html2rss/scrape_target.rb +3 -1
- data/lib/html2rss/selector_candidates.rb +234 -0
- data/lib/html2rss/selectors/README.md +64 -0
- data/lib/html2rss/selectors/extractors/attribute.rb +13 -15
- data/lib/html2rss/selectors/extractors/href.rb +15 -16
- data/lib/html2rss/selectors/extractors/html.rb +8 -9
- data/lib/html2rss/selectors/extractors/static.rb +12 -14
- data/lib/html2rss/selectors/extractors/text.rb +8 -9
- data/lib/html2rss/selectors/extractors.rb +33 -20
- data/lib/html2rss/selectors/item_env.rb +31 -0
- data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb} +11 -12
- data/lib/html2rss/selectors/option_spec.rb +140 -0
- data/lib/html2rss/selectors/post_processors/base.rb +79 -31
- data/lib/html2rss/selectors/post_processors/gsub.rb +103 -39
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +4 -4
- data/lib/html2rss/selectors/post_processors/markdown_to_html.rb +10 -11
- data/lib/html2rss/selectors/post_processors/parse_time.rb +50 -26
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +11 -7
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +46 -32
- data/lib/html2rss/selectors/post_processors/substring.rb +14 -25
- data/lib/html2rss/selectors/post_processors/template.rb +23 -19
- data/lib/html2rss/selectors/post_processors.rb +6 -5
- data/lib/html2rss/selectors/{schema_doc.rb → schema_export.rb} +12 -31
- data/lib/html2rss/selectors/step_env.rb +24 -0
- data/lib/html2rss/selectors.rb +231 -142
- data/lib/html2rss/sst/attrs.rb +5 -3
- data/lib/html2rss/sst/document.rb +3 -1
- data/lib/html2rss/sst/index.rb +3 -1
- data/lib/html2rss/sst/node.rb +29 -10
- data/lib/html2rss/sst/normalizer.rb +12 -16
- data/lib/html2rss/sst/tags.rb +7 -7
- data/lib/html2rss/status.rb +13 -8
- data/lib/html2rss/surface_category.rb +14 -32
- data/lib/html2rss/syndication/parser.rb +2 -2
- data/lib/html2rss/test/enhance_audit.rb +1 -3
- data/lib/html2rss/test/policy.rb +106 -0
- data/lib/html2rss/test.rb +126 -172
- data/lib/html2rss/url.rb +25 -4
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +27 -19
- data/lib/tasks/config_schema.rake +2 -2
- data/schema/html2rss-config.schema.json +11 -8
- metadata +19 -44
- data/lib/html2rss/request_service/faraday_strategy.rb +0 -233
- data/lib/html2rss/selectors/item_scope.rb +0 -30
|
@@ -4,8 +4,8 @@ module Html2rss
|
|
|
4
4
|
class AutoSource
|
|
5
5
|
class Segmenter
|
|
6
6
|
##
|
|
7
|
-
# Builds repeated-list article segments
|
|
8
|
-
#
|
|
7
|
+
# Builds repeated-list article segments by grouping cards under a shared parent.
|
|
8
|
+
# When the document has a +main+ or +[role=main]+, anchors outside it are dropped.
|
|
9
9
|
module List
|
|
10
10
|
module_function
|
|
11
11
|
|
|
@@ -30,40 +30,55 @@ module Html2rss
|
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def article_pairs(segmenter)
|
|
33
|
-
|
|
34
|
-
top_paths(segmenter, by_path).flat_map do |path|
|
|
35
|
-
by_path.fetch(path, []).filter_map do |node|
|
|
36
|
-
article_tag = parent_until_boundary(segmenter, node)
|
|
37
|
-
next unless article_tag
|
|
38
|
-
|
|
39
|
-
[article_tag, node]
|
|
40
|
-
end
|
|
41
|
-
end
|
|
33
|
+
top_groups(segmenter).flatten(1)
|
|
42
34
|
end
|
|
43
35
|
module_function :article_pairs
|
|
44
36
|
private_class_method :article_pairs
|
|
45
37
|
|
|
46
|
-
def
|
|
47
|
-
|
|
48
|
-
|
|
38
|
+
def top_groups(segmenter)
|
|
39
|
+
cards_by_parent(segmenter)
|
|
40
|
+
.select { |_parent, cards| cards.size >= segmenter.minimum_selector_frequency }
|
|
41
|
+
.max_by(segmenter.use_top_selectors) { |_parent, cards| cards.size }
|
|
42
|
+
.map(&:last)
|
|
43
|
+
end
|
|
44
|
+
module_function :top_groups
|
|
45
|
+
private_class_method :top_groups
|
|
46
|
+
|
|
47
|
+
def cards_by_parent(segmenter) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
48
|
+
index = segmenter.index
|
|
49
|
+
main = main_landmark(index)
|
|
50
|
+
groups = {}.compare_by_identity
|
|
51
|
+
|
|
52
|
+
index.each_node do |node|
|
|
49
53
|
next unless node.link?
|
|
50
|
-
next if
|
|
54
|
+
next if index.ignored_chrome?(node)
|
|
51
55
|
next unless relevant_anchor?(segmenter, node)
|
|
56
|
+
next if main && !inside_main?(index, node, main)
|
|
57
|
+
|
|
58
|
+
card = parent_until_boundary(segmenter, node)
|
|
59
|
+
next unless card
|
|
52
60
|
|
|
53
|
-
|
|
61
|
+
parent = index.parent_of(card)
|
|
62
|
+
next unless parent
|
|
63
|
+
|
|
64
|
+
(groups[parent] ||= []) << [card, node]
|
|
54
65
|
end
|
|
55
|
-
|
|
66
|
+
groups
|
|
67
|
+
end
|
|
68
|
+
module_function :cards_by_parent
|
|
69
|
+
private_class_method :cards_by_parent
|
|
70
|
+
|
|
71
|
+
def main_landmark(index)
|
|
72
|
+
index.each_node.find { |node| node.name == :main || node.attrs.raw['role'] == 'main' }
|
|
56
73
|
end
|
|
57
|
-
module_function :
|
|
58
|
-
private_class_method :
|
|
74
|
+
module_function :main_landmark
|
|
75
|
+
private_class_method :main_landmark
|
|
59
76
|
|
|
60
|
-
def
|
|
61
|
-
|
|
62
|
-
.max_by(segmenter.use_top_selectors) { |_path, nodes| nodes.size }
|
|
63
|
-
.map(&:first)
|
|
77
|
+
def inside_main?(index, node, main)
|
|
78
|
+
node.equal?(main) || index.descendant_of?(node, main)
|
|
64
79
|
end
|
|
65
|
-
module_function :
|
|
66
|
-
private_class_method :
|
|
80
|
+
module_function :inside_main?
|
|
81
|
+
private_class_method :inside_main?
|
|
67
82
|
|
|
68
83
|
def relevant_anchor?(segmenter, node)
|
|
69
84
|
facts = segmenter.link_resolver.destination_facts(node)
|
|
@@ -77,12 +92,15 @@ module Html2rss
|
|
|
77
92
|
module_function :relevant_anchor?
|
|
78
93
|
private_class_method :relevant_anchor?
|
|
79
94
|
|
|
95
|
+
BOUNDARY_TAGS = Set[:body, :html].freeze
|
|
96
|
+
private_constant :BOUNDARY_TAGS
|
|
97
|
+
|
|
80
98
|
def parent_until_boundary(segmenter, node)
|
|
81
99
|
index = segmenter.index
|
|
82
|
-
link_counts = Hash.new { |hash, curr| hash[curr] =
|
|
100
|
+
link_counts = Hash.new { |hash, curr| hash[curr] = count_relevant_links(segmenter, curr) }
|
|
83
101
|
|
|
84
102
|
index.parent_until(node, lambda { |curr|
|
|
85
|
-
return true if
|
|
103
|
+
return true if BOUNDARY_TAGS.include?(curr.name)
|
|
86
104
|
return false if index.ignored_chrome?(curr)
|
|
87
105
|
|
|
88
106
|
parent = index.parent_of(curr)
|
|
@@ -92,11 +110,16 @@ module Html2rss
|
|
|
92
110
|
module_function :parent_until_boundary
|
|
93
111
|
private_class_method :parent_until_boundary
|
|
94
112
|
|
|
95
|
-
def
|
|
96
|
-
|
|
113
|
+
def count_relevant_links(segmenter, node)
|
|
114
|
+
# Chrome inside a card must not stop the walk before the shared list parent.
|
|
115
|
+
if node.link?
|
|
116
|
+
relevant_anchor?(segmenter, node) ? 1 : 0
|
|
117
|
+
else
|
|
118
|
+
node.count_descendants { |child| child.link? && relevant_anchor?(segmenter, child) }
|
|
119
|
+
end
|
|
97
120
|
end
|
|
98
|
-
module_function :
|
|
99
|
-
private_class_method :
|
|
121
|
+
module_function :count_relevant_links
|
|
122
|
+
private_class_method :count_relevant_links
|
|
100
123
|
end
|
|
101
124
|
end
|
|
102
125
|
end
|
|
@@ -7,7 +7,7 @@ module Html2rss
|
|
|
7
7
|
#
|
|
8
8
|
# Strategies:
|
|
9
9
|
# - +:semantic+ — leaf article/section/li/tr/div containers with a primary link
|
|
10
|
-
# - +:list+ — repeated
|
|
10
|
+
# - +:list+ — repeated cards grouped by parent (`main` is a filter when present)
|
|
11
11
|
# - +:cluster+ — class / structure clustering for anchorless card grids
|
|
12
12
|
class Segmenter
|
|
13
13
|
# @param document [SST::Document]
|
|
@@ -68,11 +68,11 @@ module Html2rss
|
|
|
68
68
|
# @param container [SST::Node]
|
|
69
69
|
# @return [Boolean]
|
|
70
70
|
def landmark_ancestor?(anchor, container)
|
|
71
|
-
cache = (@landmark_cache ||= {})
|
|
72
|
-
|
|
73
|
-
return
|
|
71
|
+
cache = (@landmark_cache ||= {}.compare_by_identity)
|
|
72
|
+
anchor_cache = (cache[anchor] ||= {}.compare_by_identity)
|
|
73
|
+
return anchor_cache[container] if anchor_cache.key?(container)
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
anchor_cache[container] = compute_landmark_ancestor?(anchor:, container:)
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
private
|
data/lib/html2rss/auto_source.rb
CHANGED
|
@@ -93,7 +93,7 @@ module Html2rss
|
|
|
93
93
|
Scraper::SCRAPERS.sum do |scraper|
|
|
94
94
|
if config.dig(:scraper, scraper.options_key, :enabled)
|
|
95
95
|
opts = config.dig(:scraper, scraper.options_key)
|
|
96
|
-
scraper.
|
|
96
|
+
scraper.request_slots(opts)
|
|
97
97
|
else
|
|
98
98
|
0
|
|
99
99
|
end
|
|
@@ -180,7 +180,7 @@ module Html2rss
|
|
|
180
180
|
captured_responses:
|
|
181
181
|
)
|
|
182
182
|
next unless instance
|
|
183
|
-
next unless
|
|
183
|
+
next unless instance.extractable?
|
|
184
184
|
|
|
185
185
|
matched = true
|
|
186
186
|
articles.concat(run_scraper(instance))
|
data/lib/html2rss/batch.rb
CHANGED
|
@@ -10,8 +10,10 @@ module Html2rss
|
|
|
10
10
|
# Default worker threads for batch operations.
|
|
11
11
|
DEFAULT_CONCURRENCY = 5
|
|
12
12
|
|
|
13
|
+
BatchResult = Data.define(:total, :successful, :results)
|
|
14
|
+
|
|
13
15
|
# Value object representing the outcome of a batch operation.
|
|
14
|
-
BatchResult
|
|
16
|
+
class BatchResult
|
|
15
17
|
##
|
|
16
18
|
# @return [Hash{Symbol => Object}]
|
|
17
19
|
def to_h
|
|
@@ -77,7 +79,7 @@ module Html2rss
|
|
|
77
79
|
# Scrapes multiple URLs in parallel with per-URL error isolation.
|
|
78
80
|
#
|
|
79
81
|
# @param urls [Enumerable<String>] list of URLs to scrape
|
|
80
|
-
# @param strategy [Symbol, String] request strategy (+:auto+, +:
|
|
82
|
+
# @param strategy [Symbol, String] request strategy (+:auto+, +:default+, +:botasaurus+)
|
|
81
83
|
# @param limit [Integer] max articles to extract per URL
|
|
82
84
|
# @param concurrency [Integer] number of worker threads (default 5, max 10)
|
|
83
85
|
# @return [BatchResult]
|
|
@@ -89,7 +91,7 @@ module Html2rss
|
|
|
89
91
|
# Inspects multiple URLs in parallel with per-URL error isolation.
|
|
90
92
|
#
|
|
91
93
|
# @param urls [Enumerable<String>] list of URLs to inspect
|
|
92
|
-
# @param strategy [Symbol, String] request strategy (+:auto+, +:
|
|
94
|
+
# @param strategy [Symbol, String] request strategy (+:auto+, +:default+, +:botasaurus+)
|
|
93
95
|
# @param concurrency [Integer] number of worker threads (default 5, max 10)
|
|
94
96
|
# @return [BatchResult]
|
|
95
97
|
def batch_inspect(urls:, strategy: :auto, concurrency: DEFAULT_CONCURRENCY)
|
|
@@ -104,7 +106,7 @@ module Html2rss
|
|
|
104
106
|
# Runs recon across multiple URLs in parallel with per-URL error isolation.
|
|
105
107
|
#
|
|
106
108
|
# @param urls [Enumerable<String>] list of URLs to recon
|
|
107
|
-
# @param strategy [Symbol, String] request strategy (+:auto+, +:
|
|
109
|
+
# @param strategy [Symbol, String] request strategy (+:auto+, +:default+, +:botasaurus+)
|
|
108
110
|
# @param concurrency [Integer] number of worker threads (default 5, max 10)
|
|
109
111
|
# @option options [String, nil] :cache_dir optional HTML cache directory
|
|
110
112
|
# @return [BatchResult]
|
|
@@ -27,7 +27,9 @@ File.write('my-feed.yml', result.yaml)
|
|
|
27
27
|
rss = Html2rss.apply(result.config)
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
-
`Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`).
|
|
30
|
+
`Html2rss.capture` and `Capture.build` both return a `CaptureResult` with YAML (`#yaml`, includes the schema modeline) and quality meta (`has_selectors`, `segment_strategy`, `admission_drops`, `selected_strategy`, `native_feed`, `candidates`).
|
|
31
|
+
|
|
32
|
+
`#candidates` is ranked discovery evidence: `{ items:, title:, link:, published: }` arrays (best-first, deduplicated). Items entries are `{ selector:, enhance:, sample: }`; field entries are `{ selector:, sample: }`. `sample` stays on `#candidates` only — emitted YAML is still `items` + `enhance`. Buckets may be empty when evidence is missing. The default `a[href]` YAML fallback is not listed as discovered evidence.
|
|
31
33
|
|
|
32
34
|
**Wire vs internal:** user-facing **`Html2rss.apply`** ships RSS from a config Hash. Pipeline internals **`Html2rss.feed` / `feed_result`** stay unchanged — use `apply` in CLI/MCP/docs examples, not `feed`.
|
|
33
35
|
|
|
@@ -40,7 +42,7 @@ Html2rss.capture('https://example.com', strategy: :local_file, local_file_path:
|
|
|
40
42
|
Html2rss.capture('https://example.com', max_redirects: 8, max_requests: 4)
|
|
41
43
|
```
|
|
42
44
|
|
|
43
|
-
`strategy: :auto` uses the same AutoFallback chain as scrape (`
|
|
45
|
+
`strategy: :auto` uses the same AutoFallback chain as scrape (`default` → `botasaurus`). When AutoFallback selects a concrete strategy (or you pin one), Capture **stamps** `strategy:` into the emitted config so later `Html2rss.apply(config)` replays the same transport.
|
|
44
46
|
|
|
45
47
|
## CLI
|
|
46
48
|
|
|
@@ -58,13 +60,13 @@ html2rss capture https://example.com --explain # quality JSON on stderr; YAML
|
|
|
58
60
|
1. **Request** — `FeedPipeline` (AutoFallback when `:auto`)
|
|
59
61
|
2. **Discover** — AutoSource extracts admitted articles
|
|
60
62
|
3. **Segment** — try SST Segmenter strategies `:list` → `:cluster` → `:semantic`
|
|
61
|
-
4. **Gate** —
|
|
62
|
-
5. **Assemble** — `{ items: { selector:, enhance:
|
|
63
|
+
4. **Gate** — keep a derived items selector when ≥ `MIN_SELECTOR_MATCHES` (2) articles match
|
|
64
|
+
5. **Assemble** — `{ items: { selector:, enhance: } }` plus channel. HTML derivation misses fall back to `Selectors::DEFAULT_ITEMS_SELECTOR` (`segment_strategy: :default`) instead of omitting selectors.
|
|
63
65
|
|
|
64
66
|
## Constraints
|
|
65
67
|
|
|
66
68
|
- Selector quality depends on page structure; treat output as a first draft.
|
|
67
|
-
- When
|
|
69
|
+
- When HTML derivation misses (no articles, no SST document, fewer than `MIN_SELECTOR_MATCHES` pairs, or a rescued `ArgumentError`), Capture emits the default items selector with `segment_strategy: :default`. `enhance` still follows admission evidence (false when chrome drops are high). Non-HTML responses still omit selectors.
|
|
68
70
|
- Botasaurus hops need `BOTASAURUS_SCRAPER_URL`.
|
|
69
71
|
|
|
70
72
|
See also {Html2rss::AutoSource} for article discovery and {Html2rss::FeedPipeline} for the `:auto` request chain.
|
data/lib/html2rss/capture.rb
CHANGED
|
@@ -25,12 +25,15 @@ module Html2rss
|
|
|
25
25
|
CHROME_DROP_THRESHOLD = 3
|
|
26
26
|
private_constant :CHROME_DROP_THRESHOLD
|
|
27
27
|
|
|
28
|
-
##
|
|
29
|
-
# Result of a capture operation (config plus quality meta).
|
|
30
28
|
CaptureResult = Data.define(
|
|
31
29
|
:config, :yaml, :articles_count, :channel_title, :has_selectors, :segment_strategy,
|
|
32
|
-
:admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url
|
|
33
|
-
|
|
30
|
+
:admission_drops, :selected_strategy, :inferred_topics, :native_feed, :suggested_channel_url,
|
|
31
|
+
:candidates
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
##
|
|
35
|
+
# Result of a capture operation (config plus quality meta).
|
|
36
|
+
class CaptureResult
|
|
34
37
|
# rubocop:disable Metrics/ParameterLists
|
|
35
38
|
##
|
|
36
39
|
# @param config [Hash]
|
|
@@ -44,9 +47,10 @@ module Html2rss
|
|
|
44
47
|
# @param inferred_topics [Array<String>]
|
|
45
48
|
# @param native_feed [String, nil]
|
|
46
49
|
# @param suggested_channel_url [String, nil]
|
|
50
|
+
# @param candidates [Hash{Symbol=>Array}, nil] ranked selector buckets from {SelectorCandidates}
|
|
47
51
|
def initialize(config:, articles_count:, channel_title:, has_selectors:, segment_strategy:, # rubocop:disable Metrics/MethodLength
|
|
48
52
|
yaml: nil, admission_drops: {}, selected_strategy: nil, inferred_topics: [], native_feed: nil,
|
|
49
|
-
suggested_channel_url: nil)
|
|
53
|
+
suggested_channel_url: nil, candidates: nil)
|
|
50
54
|
super(
|
|
51
55
|
config:,
|
|
52
56
|
yaml: yaml || "#{SCHEMA_MODELINE}\n#{Config.to_yaml(config)}",
|
|
@@ -58,7 +62,8 @@ module Html2rss
|
|
|
58
62
|
selected_strategy:,
|
|
59
63
|
inferred_topics:,
|
|
60
64
|
native_feed:,
|
|
61
|
-
suggested_channel_url
|
|
65
|
+
suggested_channel_url:,
|
|
66
|
+
candidates: candidates || SelectorCandidates.empty
|
|
62
67
|
)
|
|
63
68
|
end
|
|
64
69
|
# rubocop:enable Metrics/ParameterLists
|
|
@@ -75,7 +80,7 @@ module Html2rss
|
|
|
75
80
|
# Analyzes a URL and builds a reusable feed config.
|
|
76
81
|
#
|
|
77
82
|
# @param url [String] source page URL
|
|
78
|
-
# @param strategy [Symbol] request strategy (+:auto+, +:
|
|
83
|
+
# @param strategy [Symbol] request strategy (+:auto+, +:default+, +:botasaurus+)
|
|
79
84
|
# @option options [String, nil] :items_selector optional selector hint
|
|
80
85
|
# @option options [Array<String>, nil] :topics optional directory topics override
|
|
81
86
|
# @option options [String, nil] :title optional title override
|
|
@@ -127,7 +132,7 @@ module Html2rss
|
|
|
127
132
|
def build # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
128
133
|
outcome = FeedPipeline.new(raw_config).to_outcome
|
|
129
134
|
@admission_drops = outcome.admission_drops
|
|
130
|
-
selectors, segment_strategy = derive_selectors(outcome.response, outcome.articles)
|
|
135
|
+
selectors, segment_strategy, candidates = derive_selectors(outcome.response, outcome.articles)
|
|
131
136
|
ch_title = @title || channel_title_from(outcome.response)
|
|
132
137
|
topics = @topics || infer_topics("#{@url} #{ch_title}")
|
|
133
138
|
native_feed = probe_native_feed(outcome.response) unless @force
|
|
@@ -151,7 +156,8 @@ module Html2rss
|
|
|
151
156
|
selected_strategy: outcome.selected_strategy,
|
|
152
157
|
inferred_topics: topics,
|
|
153
158
|
native_feed:,
|
|
154
|
-
suggested_channel_url: suggested_channel_url(outcome)
|
|
159
|
+
suggested_channel_url: suggested_channel_url(outcome),
|
|
160
|
+
candidates:
|
|
155
161
|
)
|
|
156
162
|
end
|
|
157
163
|
|
|
@@ -254,40 +260,68 @@ module Html2rss
|
|
|
254
260
|
{ strategy: :local_file, request: { local_file_path: @local_file_path } }
|
|
255
261
|
end
|
|
256
262
|
|
|
257
|
-
# @return [Array(Hash, Symbol, nil)] selectors
|
|
263
|
+
# @return [Array(Hash, Symbol, nil, Hash)] selectors, segment strategy, candidate buckets
|
|
258
264
|
def derive_selectors(response, articles)
|
|
259
|
-
return hint_selectors if @items_selector_hint
|
|
260
|
-
return [{}, nil
|
|
265
|
+
return [*hint_selectors, SelectorCandidates.empty] if @items_selector_hint
|
|
266
|
+
return [{}, nil, SelectorCandidates.empty] unless response.html_response?
|
|
267
|
+
return [*default_selectors, SelectorCandidates.empty] if articles.empty?
|
|
261
268
|
|
|
262
269
|
sst = SST::Normalizer.call(response.body)
|
|
263
|
-
return [
|
|
270
|
+
return [*default_selectors, SelectorCandidates.empty] unless sst
|
|
264
271
|
|
|
265
272
|
select_enhance_selectors(sst, articles)
|
|
266
273
|
rescue ArgumentError => error
|
|
267
274
|
Log.warn("Capture selector derivation failed: #{error.message}")
|
|
268
|
-
[
|
|
275
|
+
[*default_selectors, SelectorCandidates.empty]
|
|
269
276
|
end
|
|
270
277
|
|
|
271
278
|
def hint_selectors
|
|
272
279
|
[{ items: { selector: @items_selector_hint, enhance: resolve_enhance } }, :hint]
|
|
273
280
|
end
|
|
274
281
|
|
|
275
|
-
def
|
|
282
|
+
def default_selectors
|
|
283
|
+
[{ items: { selector: Selectors::DEFAULT_ITEMS_SELECTOR, enhance: resolve_enhance } }, :default]
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def select_enhance_selectors(sst, articles)
|
|
287
|
+
evidence = collect_items_evidence(sst, articles)
|
|
288
|
+
candidates = SelectorCandidates.call(items_evidence: evidence, min_matches: MIN_SELECTOR_MATCHES)
|
|
289
|
+
top = candidates[:items].first
|
|
290
|
+
return [*default_selectors, candidates] unless top
|
|
291
|
+
|
|
292
|
+
strategy = evidence.find { |entry| entry[:selector] == top[:selector] }&.fetch(:strategy)
|
|
293
|
+
items = { selector: top[:selector], enhance: top[:enhance] }
|
|
294
|
+
[{ items: }, strategy, candidates]
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def collect_items_evidence(sst, articles)
|
|
276
298
|
link_resolver = Scoring::LinkResolver.new(@url)
|
|
277
299
|
enhance = resolve_enhance
|
|
278
300
|
|
|
279
|
-
SEGMENT_STRATEGIES.
|
|
301
|
+
SEGMENT_STRATEGIES.flat_map do |strategy|
|
|
280
302
|
segments = AutoSource::Segmenter.call(
|
|
281
303
|
sst, base_url: @url, strategy:, permit_unanchored: false, link_resolver:
|
|
282
304
|
)
|
|
283
305
|
matched = match_segments_to_articles(segments, articles)
|
|
284
|
-
|
|
285
|
-
|
|
306
|
+
next [] if matched.size < MIN_SELECTOR_MATCHES
|
|
307
|
+
|
|
308
|
+
items_evidence_for(matched, strategy:, enhance:)
|
|
309
|
+
end
|
|
310
|
+
end
|
|
286
311
|
|
|
287
|
-
|
|
312
|
+
def items_evidence_for(matched, strategy:, enhance:)
|
|
313
|
+
roots = lift_heading_link_roots(matched.map { |pair| pair[:segment].root_node })
|
|
314
|
+
selector_variants_for(roots).map do |kind, selector|
|
|
315
|
+
{ selector:, enhance:, strategy:, kind:, match_count: matched.size, roots: }
|
|
288
316
|
end
|
|
317
|
+
end
|
|
289
318
|
|
|
290
|
-
|
|
319
|
+
def selector_variants_for(roots)
|
|
320
|
+
[
|
|
321
|
+
[:shared_class, shared_class_items_selector(roots)],
|
|
322
|
+
[:unique_tag, unique_tag_items_selector(roots)],
|
|
323
|
+
[:path, path_items_selector(roots)]
|
|
324
|
+
].filter_map { |kind, selector| [kind, selector] if selector }
|
|
291
325
|
end
|
|
292
326
|
|
|
293
327
|
def match_segments_to_articles(segments, articles)
|
|
@@ -322,21 +356,6 @@ module Html2rss
|
|
|
322
356
|
segment.root_node.visible_text.to_s.strip
|
|
323
357
|
end
|
|
324
358
|
|
|
325
|
-
def items_selector(matched)
|
|
326
|
-
return nil if matched.empty?
|
|
327
|
-
|
|
328
|
-
roots = lift_heading_link_roots(matched.map { |m| m[:segment].root_node })
|
|
329
|
-
shared = shared_class_items_selector(roots)
|
|
330
|
-
return shared if shared
|
|
331
|
-
|
|
332
|
-
paths = roots.map(&:tag_path)
|
|
333
|
-
common = common_path_prefix(paths)
|
|
334
|
-
tag_path = common.empty? ? paths.first.to_s : common
|
|
335
|
-
return nil if tag_path.empty?
|
|
336
|
-
|
|
337
|
-
css_from_trimmed_tag_path(tag_path)
|
|
338
|
-
end
|
|
339
|
-
|
|
340
359
|
def lift_heading_link_roots(roots)
|
|
341
360
|
roots.map { |root| lift_heading_link_root(root, roots) }
|
|
342
361
|
end
|
|
@@ -371,7 +390,6 @@ module Html2rss
|
|
|
371
390
|
|
|
372
391
|
def wrapping_anchor_root?(node)
|
|
373
392
|
return false unless node.name.to_s == 'a'
|
|
374
|
-
return false unless node.respond_to?(:find)
|
|
375
393
|
|
|
376
394
|
tags = Html::Navigator::WRAPPING_ANCHOR_CHILD_TAGS
|
|
377
395
|
node.find { |child| !child.equal?(node) && tags.include?(child.name.to_s) }
|
|
@@ -396,6 +414,20 @@ module Html2rss
|
|
|
396
414
|
"#{roots.first.name}.#{shared.min}"
|
|
397
415
|
end
|
|
398
416
|
|
|
417
|
+
def unique_tag_items_selector(roots)
|
|
418
|
+
names = roots.map { |root| root.name.to_s }.uniq
|
|
419
|
+
names.join(', ') if names.size > 1
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
def path_items_selector(roots)
|
|
423
|
+
paths = roots.map(&:tag_path)
|
|
424
|
+
common = common_path_prefix(paths)
|
|
425
|
+
tag_path = common.empty? ? paths.first.to_s : common
|
|
426
|
+
return if tag_path.empty?
|
|
427
|
+
|
|
428
|
+
css_from_trimmed_tag_path(tag_path)
|
|
429
|
+
end
|
|
430
|
+
|
|
399
431
|
def css_from_trimmed_tag_path(tag_path)
|
|
400
432
|
segments = tag_path.split('/').reject(&:empty?)
|
|
401
433
|
trimmed = trim_leading_items_segments(segments)
|
data/lib/html2rss/channel.rb
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require 'time'
|
|
4
4
|
|
|
5
|
-
# rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
|
|
6
5
|
module Html2rss
|
|
6
|
+
Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author)
|
|
7
|
+
|
|
7
8
|
##
|
|
8
9
|
# Materialized channel metadata for feed builders and {FeedResult}.
|
|
9
10
|
#
|
|
@@ -27,7 +28,7 @@ module Html2rss
|
|
|
27
28
|
# @return [Html2rss::Url, nil]
|
|
28
29
|
# @!attribute [r] author
|
|
29
30
|
# @return [String, nil]
|
|
30
|
-
Channel
|
|
31
|
+
class Channel # rubocop:disable Metrics/ClassLength -- Channel value object owns extract + freeze + Marshal
|
|
31
32
|
class << self
|
|
32
33
|
##
|
|
33
34
|
# Materializes channel attributes from an HTTP response and optional overrides.
|
|
@@ -193,5 +194,4 @@ module Html2rss
|
|
|
193
194
|
Time.httpdate(value)
|
|
194
195
|
end
|
|
195
196
|
end
|
|
196
|
-
# rubocop:enable Metrics/ModuleLength
|
|
197
197
|
end
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
class CLI
|
|
5
|
-
##
|
|
6
|
-
# Display-only probe facts for inspect and recon text cards.
|
|
7
5
|
ProbeView = Data.define(
|
|
8
6
|
:requested,
|
|
9
7
|
:final,
|
|
@@ -15,7 +13,11 @@ module Html2rss
|
|
|
15
13
|
:alternate_feeds,
|
|
16
14
|
:notes,
|
|
17
15
|
:strategy
|
|
18
|
-
)
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
##
|
|
19
|
+
# Display-only probe facts for inspect and recon text cards.
|
|
20
|
+
class ProbeView
|
|
19
21
|
class << self
|
|
20
22
|
##
|
|
21
23
|
# @param data [Hash] inspect wire payload
|
data/lib/html2rss/cli/render.rb
CHANGED
|
@@ -93,9 +93,11 @@ module Html2rss
|
|
|
93
93
|
end
|
|
94
94
|
end
|
|
95
95
|
else
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
result.
|
|
96
|
+
$stderr.puts "\e[31m✗ Test failed\e[0m (#{source})" # rubocop:disable Style/StderrPuts
|
|
97
|
+
$stderr.puts " Error: #{result.error_message}" if result.error_message # rubocop:disable Style/StderrPuts
|
|
98
|
+
result.validation_issues&.each do |issue|
|
|
99
|
+
$stderr.puts " Schema error #{issue}" # rubocop:disable Style/StderrPuts
|
|
100
|
+
end
|
|
99
101
|
end
|
|
100
102
|
end
|
|
101
103
|
|
|
@@ -115,8 +117,7 @@ module Html2rss
|
|
|
115
117
|
if view.final && view.final != view.requested
|
|
116
118
|
puts " Final: #{view.final} (HTTP #{view.status || 'ERR'})"
|
|
117
119
|
end
|
|
118
|
-
|
|
119
|
-
puts " Surface: #{surface} (#{view.articles_count} articles)"
|
|
120
|
+
puts " Surface: #{view.surface} (#{view.articles_count} articles)"
|
|
120
121
|
end
|
|
121
122
|
private_class_method :probe_lines
|
|
122
123
|
end
|
|
@@ -29,11 +29,10 @@ module Html2rss
|
|
|
29
29
|
if result.success?
|
|
30
30
|
puts(target_files.size == 1 ? 'Configuration is valid' : "ok #{file}") unless quiet
|
|
31
31
|
else
|
|
32
|
-
|
|
33
|
-
raise Thor::Error, "Invalid configuration: #{error_details}" if target_files.size == 1
|
|
32
|
+
raise Thor::Error, format_failure(result) if target_files.size == 1
|
|
34
33
|
|
|
35
34
|
warn "FAIL #{file}"
|
|
36
|
-
|
|
35
|
+
result.issues.each { |issue| warn " #{issue}" }
|
|
37
36
|
failed << file
|
|
38
37
|
end
|
|
39
38
|
end
|
|
@@ -57,35 +56,37 @@ module Html2rss
|
|
|
57
56
|
resolved.uniq
|
|
58
57
|
end
|
|
59
58
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
|
|
59
|
+
class << self
|
|
60
|
+
private
|
|
63
61
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
62
|
+
def run_named_feed(file, feed_name, params:, quiet:)
|
|
63
|
+
result = Html2rss.validate(file, feed_name, params:)
|
|
64
|
+
raise Thor::Error, format_failure(result) unless result.success?
|
|
65
|
+
|
|
66
|
+
puts 'Configuration is valid' unless quiet
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def validate_file(file, params:)
|
|
70
|
+
if file == '-'
|
|
71
|
+
Html2rss.validate($stdin.read, params:)
|
|
72
|
+
else
|
|
73
|
+
Html2rss.validate(file, params:)
|
|
74
|
+
end
|
|
74
75
|
end
|
|
75
|
-
end
|
|
76
|
-
module_function :validate_file
|
|
77
|
-
private_class_method :validate_file
|
|
78
76
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
return true if path.end_with?('.yml', '.yaml')
|
|
83
|
-
return true if path.include?('/') || path.include?('*')
|
|
77
|
+
def format_failure(report)
|
|
78
|
+
"Invalid configuration: #{report}"
|
|
79
|
+
end
|
|
84
80
|
|
|
85
|
-
|
|
81
|
+
def path_like_config?(arg)
|
|
82
|
+
path = arg.to_s
|
|
83
|
+
return true if path.start_with?('-')
|
|
84
|
+
return true if path.end_with?('.yml', '.yaml')
|
|
85
|
+
return true if path.include?('/') || path.include?('*')
|
|
86
|
+
|
|
87
|
+
false
|
|
88
|
+
end
|
|
86
89
|
end
|
|
87
|
-
module_function :path_like_config?
|
|
88
|
-
private_class_method :path_like_config?
|
|
89
90
|
end
|
|
90
91
|
end
|
|
91
92
|
end
|
data/lib/html2rss/cli.rb
CHANGED
|
@@ -300,7 +300,7 @@ module Html2rss
|
|
|
300
300
|
def filter_recon_results(results, verdict_filter)
|
|
301
301
|
return results unless verdict_filter
|
|
302
302
|
|
|
303
|
-
expected = Recon::Verdict
|
|
303
|
+
expected = Recon::Verdict[verdict_filter.downcase]
|
|
304
304
|
results.select { |r| r.verdict == expected }
|
|
305
305
|
rescue ArgumentError => error
|
|
306
306
|
raise Thor::Error, error.message
|