html2rss 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/html2rss/article.rb +42 -35
- data/lib/html2rss/auto_source/scraper/html.rb +68 -145
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +116 -0
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +63 -0
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +179 -0
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +55 -0
- data/lib/html2rss/auto_source/scraper/json_state.rb +0 -379
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +29 -114
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +50 -11
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +14 -5
- data/lib/html2rss/auto_source/scraper/schema.rb +61 -18
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +35 -30
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +40 -120
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +161 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +10 -7
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +6 -2
- data/lib/html2rss/auto_source/scraper.rb +81 -44
- data/lib/html2rss/auto_source/segment.rb +29 -0
- data/lib/html2rss/auto_source/segmenter/cluster.rb +163 -0
- data/lib/html2rss/auto_source/segmenter/list.rb +101 -0
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +81 -0
- data/lib/html2rss/auto_source/segmenter/semantic.rb +104 -0
- data/lib/html2rss/auto_source/segmenter.rb +66 -0
- data/lib/html2rss/auto_source.rb +74 -21
- data/lib/html2rss/channel.rb +159 -79
- data/lib/html2rss/cli.rb +8 -13
- data/lib/html2rss/config/auto_source_contract.rb +2 -0
- data/lib/html2rss/config/request_controls.rb +33 -0
- data/lib/html2rss/config/validator.rb +15 -8
- data/lib/html2rss/config.rb +6 -2
- data/lib/html2rss/feed_builder/item_presentation.rb +42 -0
- data/lib/html2rss/feed_builder/json_feed/item.rb +20 -4
- data/lib/html2rss/feed_builder/json_feed.rb +22 -2
- data/lib/html2rss/feed_builder/rss.rb +36 -24
- data/lib/html2rss/feed_builder.rb +9 -17
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +71 -28
- data/lib/html2rss/feed_pipeline.rb +66 -55
- data/lib/html2rss/feed_result.rb +92 -0
- data/lib/html2rss/html/article_extractor/category_extractor.rb +10 -36
- data/lib/html2rss/html/article_extractor/date_extractor.rb +2 -7
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +5 -50
- data/lib/html2rss/html/article_extractor/image_extractor.rb +5 -16
- data/lib/html2rss/html/article_rules/category.rb +55 -0
- data/lib/html2rss/html/article_rules/date.rb +25 -0
- data/lib/html2rss/html/article_rules/enclosure.rb +72 -0
- data/lib/html2rss/html/article_rules/image.rb +109 -0
- data/lib/html2rss/html/article_rules.rb +10 -0
- data/lib/html2rss/html/rendering/audio_renderer.rb +2 -12
- data/lib/html2rss/html/rendering/escaped_attributes.rb +27 -0
- data/lib/html2rss/html/rendering/image_renderer.rb +2 -12
- data/lib/html2rss/html/rendering/pdf_renderer.rb +2 -8
- data/lib/html2rss/html/rendering/video_renderer.rb +2 -12
- data/lib/html2rss/html/sst_article_extractor.rb +279 -0
- data/lib/html2rss/link_destination/destination_facts.rb +40 -0
- data/lib/html2rss/link_destination/noise_policy.rb +79 -0
- data/lib/html2rss/link_destination/path_classifier.rb +205 -0
- data/lib/html2rss/link_destination/text_classifier.rb +64 -0
- data/lib/html2rss/link_destination.rb +8 -0
- data/lib/html2rss/request_service/botasaurus_contract.rb +33 -5
- data/lib/html2rss/request_service/botasaurus_strategy.rb +13 -4
- data/lib/html2rss/request_service/budget.rb +7 -2
- data/lib/html2rss/request_service/faraday_strategy.rb +42 -1
- data/lib/html2rss/request_service/network_guard.rb +5 -3
- data/lib/html2rss/request_service/response.rb +9 -1
- data/lib/html2rss/request_service/strategy.rb +22 -0
- data/lib/html2rss/request_session.rb +4 -0
- data/lib/html2rss/scoring/anchor_score.rb +34 -0
- data/lib/html2rss/scoring/cluster_scorer.rb +87 -0
- data/lib/html2rss/scoring/container_assessor.rb +83 -0
- data/lib/html2rss/scoring/engine.rb +101 -0
- data/lib/html2rss/scoring/link_resolver.rb +72 -0
- data/lib/html2rss/scoring/observation.rb +53 -0
- data/lib/html2rss/scoring/ranked_segment.rb +45 -0
- data/lib/html2rss/scoring/score.rb +30 -0
- data/lib/html2rss/scoring.rb +33 -0
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +10 -1
- data/lib/html2rss/selectors.rb +26 -24
- data/lib/html2rss/sst/attrs.rb +91 -0
- data/lib/html2rss/sst/document.rb +23 -0
- data/lib/html2rss/sst/index.rb +112 -0
- data/lib/html2rss/sst/node.rb +147 -0
- data/lib/html2rss/sst/normalizer.rb +171 -0
- data/lib/html2rss/sst/tags.rb +26 -0
- data/lib/html2rss/sst/text.rb +81 -0
- data/lib/html2rss/sst.rb +8 -0
- data/lib/html2rss/status.rb +132 -0
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +41 -29
- data/schema/html2rss-config.schema.json +20 -0
- metadata +45 -19
- data/lib/html2rss/auto_source/discovery/dom_clustering/group_scorer.rb +0 -80
- data/lib/html2rss/auto_source/discovery/dom_clustering/overlap_resolver.rb +0 -86
- data/lib/html2rss/auto_source/discovery/dom_clustering.rb +0 -119
- data/lib/html2rss/auto_source/discovery/list_candidates.rb +0 -94
- data/lib/html2rss/auto_source/discovery/semantic_anchor_candidates.rb +0 -219
- data/lib/html2rss/auto_source/discovery/semantic_containers.rb +0 -71
- data/lib/html2rss/auto_source/discovery/sitemap.rb +0 -159
- data/lib/html2rss/auto_source/discovery.rb +0 -14
- data/lib/html2rss/auto_source/link_heuristics/anchor_signals.rb +0 -28
- data/lib/html2rss/auto_source/link_heuristics/container_assessor.rb +0 -106
- data/lib/html2rss/auto_source/link_heuristics/container_signals.rb +0 -80
- data/lib/html2rss/auto_source/link_heuristics/destination_facts.rb +0 -42
- data/lib/html2rss/auto_source/link_heuristics/href_extractor.rb +0 -38
- data/lib/html2rss/auto_source/link_heuristics/path_classifier.rb +0 -221
- data/lib/html2rss/auto_source/link_heuristics/text_classifier.rb +0 -66
- data/lib/html2rss/auto_source/link_heuristics.rb +0 -139
- data/lib/html2rss/auto_source/scraper/schema/list_item.rb +0 -28
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1b6b0ddc57b5cf372cc9bc6405bb21f7dc6d00aa526e234132974ccc6ef609f7
|
|
4
|
+
data.tar.gz: 59a60ef044a99686838b5498f0712a455a6c5a832c27619db0c99ba21742f6a8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8994aa3b90d791ea8cf75df9b576669fffe505fe0895e207745ab0e591f187ab490382d00a95d49fc1073570f886d6940f27f1c1ca80ba7c57817740f48c4bc4
|
|
7
|
+
data.tar.gz: dc62483a0ade2b582fb3839778a3a518a7f1a08e781d2105c0e93c70aaa26313e153a4e30121170b8a59e8472517e6cb8d9958b9c06fa3a862358631f469c18a
|
data/README.md
CHANGED
|
@@ -13,7 +13,7 @@ Most people looking for a first working feed should start with `html2rss-web`, r
|
|
|
13
13
|
Detailed usage guides, reference docs, and the feed directory live on the project website:
|
|
14
14
|
|
|
15
15
|
- [Ruby gem documentation](https://html2rss.github.io/ruby-gem)
|
|
16
|
-
- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto
|
|
16
|
+
- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto` = `faraday` → `botasaurus`; pin `browserless` explicitly)
|
|
17
17
|
- [Selectors & pagination](https://html2rss.github.io/ruby-gem/reference/selectors#paginated-feeds)
|
|
18
18
|
- [Web application](https://html2rss.github.io/web-application)
|
|
19
19
|
- [Feed directory](https://html2rss.github.io/feed-directory)
|
|
@@ -26,7 +26,7 @@ Cloud development: [Open in GitHub Codespaces](https://github.com/codespaces/new
|
|
|
26
26
|
## Architecture
|
|
27
27
|
|
|
28
28
|
1. **Config** — loads and validates configuration (YAML/hash); schema via `html2rss schema` / `schema/html2rss-config.schema.json`
|
|
29
|
-
2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or `browserless`)
|
|
29
|
+
2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or explicit `browserless`)
|
|
30
30
|
3. **Selectors** — extracts content via CSS selectors with extractors/post-processors
|
|
31
31
|
4. **AutoSource** — auto-detects content (Schema.org, JSON state, semantic HTML, structural patterns)
|
|
32
32
|
5. **FeedBuilder** — assembles Article objects and renders feeds (RSS 2.0 / JSON Feed 1.1)
|
data/lib/html2rss/article.rb
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'zlib'
|
|
4
|
-
require 'sanitize'
|
|
5
|
-
require 'nokogiri'
|
|
6
4
|
|
|
7
5
|
module Html2rss
|
|
8
6
|
##
|
|
9
7
|
# Article is a simple data object representing an article extracted from a page.
|
|
10
8
|
# It is enumerable and responds to all keys specified in PROVIDED_KEYS.
|
|
11
|
-
#
|
|
9
|
+
#
|
|
10
|
+
# Description and enclosure wire presentation live in {FeedBuilder::ItemPresentation}.
|
|
11
|
+
# rubocop:disable Metrics/ClassLength -- value object retains Marshal + defensive freeze helpers
|
|
12
12
|
class Article
|
|
13
13
|
include Enumerable
|
|
14
14
|
include Comparable
|
|
@@ -26,9 +26,9 @@ module Html2rss
|
|
|
26
26
|
# @param options [Hash{Symbol => String}]
|
|
27
27
|
# @option options [String] :id stable article identifier
|
|
28
28
|
# @option options [String] :title article title
|
|
29
|
-
# @option options [String] :description
|
|
29
|
+
# @option options [String] :description raw extracted description/content (not feed-rendered HTML)
|
|
30
30
|
# @option options [String, Html2rss::Url] :url canonical article URL
|
|
31
|
-
# @option options [String, Html2rss::Url] :image image URL for
|
|
31
|
+
# @option options [String, Html2rss::Url] :image image URL for description / JSON Feed +image+
|
|
32
32
|
# @option options [String] :author author name
|
|
33
33
|
# @option options [String] :guid explicit GUID override
|
|
34
34
|
# @option options [String, Time, DateTime] :published_at publication timestamp
|
|
@@ -36,9 +36,9 @@ module Html2rss
|
|
|
36
36
|
# @option options [Array<String>] :categories category labels
|
|
37
37
|
# @option options [Class] :scraper scraper class that produced the article
|
|
38
38
|
def initialize(**options)
|
|
39
|
-
@to_h = options.each_with_object({}) { |(
|
|
39
|
+
@to_h = options.each_with_object({}) { |(key, value), hash| hash[key] = freeze_option(value) }.freeze
|
|
40
40
|
|
|
41
|
-
@
|
|
41
|
+
@url = @image = @guid = @enclosures = @categories = @published_at = NOT_SET
|
|
42
42
|
|
|
43
43
|
return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
|
|
44
44
|
|
|
@@ -65,18 +65,10 @@ module Html2rss
|
|
|
65
65
|
# @return [String, nil] article title
|
|
66
66
|
def title = blank_string_to_nil(@to_h[:title])
|
|
67
67
|
|
|
68
|
-
#
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
@description = Html::Rendering::DescriptionBuilder.new(
|
|
73
|
-
base: @to_h[:description],
|
|
74
|
-
title:,
|
|
75
|
-
url:,
|
|
76
|
-
enclosures:,
|
|
77
|
-
image:
|
|
78
|
-
).call
|
|
79
|
-
end
|
|
68
|
+
# Raw extracted description — feed HTML enrichment is {FeedBuilder::ItemPresentation.description_for}.
|
|
69
|
+
#
|
|
70
|
+
# @return [String, nil]
|
|
71
|
+
def description = blank_string_to_nil(@to_h[:description])
|
|
80
72
|
|
|
81
73
|
# @return [Url, nil]
|
|
82
74
|
def url
|
|
@@ -117,21 +109,7 @@ module Html2rss
|
|
|
117
109
|
|
|
118
110
|
@enclosures = Array(@to_h[:enclosures])
|
|
119
111
|
.map { |enclosure| Enclosure.new(**enclosure) }
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
# @return [Html2rss::Article::Enclosure, nil]
|
|
123
|
-
def enclosure
|
|
124
|
-
return @enclosure unless @enclosure == NOT_SET
|
|
125
|
-
|
|
126
|
-
@enclosure = case (object = @to_h[:enclosures]&.first)
|
|
127
|
-
when Hash
|
|
128
|
-
Enclosure.new(**object)
|
|
129
|
-
when nil
|
|
130
|
-
Enclosure.new(url: image) if image
|
|
131
|
-
else
|
|
132
|
-
Log.warn "Article: unknown enclosure type: #{object.class}"
|
|
133
|
-
nil
|
|
134
|
-
end
|
|
112
|
+
.freeze
|
|
135
113
|
end
|
|
136
114
|
|
|
137
115
|
# @return [Array<String>] normalized, unique category names
|
|
@@ -142,7 +120,7 @@ module Html2rss
|
|
|
142
120
|
categories.map! { |category| category.to_s.strip }
|
|
143
121
|
categories.reject!(&:empty?)
|
|
144
122
|
categories.uniq!
|
|
145
|
-
end
|
|
123
|
+
end.freeze
|
|
146
124
|
end
|
|
147
125
|
|
|
148
126
|
# Parses and returns the published_at time.
|
|
@@ -171,6 +149,35 @@ module Html2rss
|
|
|
171
149
|
|
|
172
150
|
private
|
|
173
151
|
|
|
152
|
+
##
|
|
153
|
+
# Marshal only the constructor payload so lazy +NOT_SET+ sentinels do not break
|
|
154
|
+
# {FeedResult} cache round-trips.
|
|
155
|
+
#
|
|
156
|
+
# @return [Hash{Symbol => Object}]
|
|
157
|
+
def marshal_dump = @to_h
|
|
158
|
+
|
|
159
|
+
##
|
|
160
|
+
# @param payload [Hash{Symbol => Object}] constructor options from {#marshal_dump}
|
|
161
|
+
# @return [void]
|
|
162
|
+
def marshal_load(payload)
|
|
163
|
+
initialize(**payload)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def freeze_option(value)
|
|
167
|
+
case value
|
|
168
|
+
when String then value.dup.freeze
|
|
169
|
+
when Array then freeze_array_option(value)
|
|
170
|
+
when Hash then value.transform_values { freeze_option(_1) }.freeze
|
|
171
|
+
else value
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def freeze_array_option(value)
|
|
176
|
+
value.map do |entry|
|
|
177
|
+
entry.is_a?(Hash) ? entry.transform_values { freeze_option(_1) }.freeze : freeze_option(entry)
|
|
178
|
+
end.freeze
|
|
179
|
+
end
|
|
180
|
+
|
|
174
181
|
def dedup_from_url
|
|
175
182
|
return unless (value = url)
|
|
176
183
|
|
|
@@ -1,21 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'nokogiri'
|
|
4
|
-
|
|
5
3
|
module Html2rss
|
|
6
4
|
class AutoSource
|
|
7
5
|
module Scraper
|
|
8
6
|
##
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
# The approach is intentionally heuristic:
|
|
13
|
-
# 1. collect repeated anchor paths
|
|
14
|
-
# 2. walk upward to a shared container shape
|
|
15
|
-
# 3. extract the best anchor found inside each container
|
|
16
|
-
#
|
|
17
|
-
# This scraper is broader and noisier than `SemanticHtml`, so it acts as a
|
|
18
|
-
# fallback for pages without stronger semantic signals.
|
|
7
|
+
# Fallback HTML list/cluster scraper via SST pipeline
|
|
8
|
+
# (Normalizer → Segmenter → Scoring::Engine → Html::SstArticleExtractor).
|
|
19
9
|
class Html
|
|
20
10
|
include Enumerable
|
|
21
11
|
|
|
@@ -25,53 +15,44 @@ module Html2rss
|
|
|
25
15
|
DEFAULT_MINIMUM_SELECTOR_FREQUENCY = 2
|
|
26
16
|
# Number of most frequent selectors kept for container extraction.
|
|
27
17
|
DEFAULT_USE_TOP_SELECTORS = 5
|
|
18
|
+
# Maximum articles materialized after eligibility filtering.
|
|
19
|
+
TOP_K = Scoring::Engine::TOP_K
|
|
28
20
|
|
|
29
21
|
##
|
|
30
|
-
# @return [Symbol]
|
|
22
|
+
# @return [Symbol]
|
|
31
23
|
def self.options_key = :html
|
|
32
24
|
|
|
33
25
|
##
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
# containers.
|
|
37
|
-
#
|
|
38
|
-
# @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
|
|
39
|
-
# @return [Boolean] true when the scraper can likely extract articles
|
|
26
|
+
# @param parsed_body [Nokogiri::HTML::Document]
|
|
27
|
+
# @return [Boolean]
|
|
40
28
|
def self.articles?(parsed_body)
|
|
41
|
-
|
|
42
|
-
end
|
|
29
|
+
return false unless parsed_body
|
|
43
30
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
#
|
|
48
|
-
# @param xpath [String] original XPath
|
|
49
|
-
# @return [String] XPath without positional indexes
|
|
50
|
-
def self.simplify_xpath(xpath)
|
|
51
|
-
Discovery::ListCandidates.simplify_xpath(xpath)
|
|
31
|
+
new(parsed_body, url: DETECTION_BASE_URL).extractable?
|
|
32
|
+
rescue ArgumentError
|
|
33
|
+
false
|
|
52
34
|
end
|
|
53
35
|
|
|
54
|
-
# @param parsed_body [Nokogiri::HTML::Document]
|
|
55
|
-
# @param url [String]
|
|
56
|
-
# @param
|
|
57
|
-
# @param
|
|
58
|
-
# @
|
|
59
|
-
# @option opts [
|
|
60
|
-
|
|
36
|
+
# @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
|
|
37
|
+
# @param url [String, Html2rss::Url]
|
|
38
|
+
# @param document [SST::Document, nil] memoized SST document from AutoSource
|
|
39
|
+
# @param link_resolver [Scoring::LinkResolver, nil] shared page-scoped resolver
|
|
40
|
+
# @param opts [Hash]
|
|
41
|
+
# @option opts [Boolean] :fallback_anchorless keep anchorless cluster cards
|
|
42
|
+
# @option opts [Integer] :minimum_selector_frequency list frequency floor
|
|
43
|
+
# @option opts [Integer] :use_top_selectors list selector budget
|
|
44
|
+
def initialize(parsed_body = nil, url:, document: nil, link_resolver: nil, **opts)
|
|
61
45
|
@parsed_body = parsed_body
|
|
46
|
+
@provided_document = document
|
|
47
|
+
@provided_link_resolver = link_resolver
|
|
62
48
|
@url = url
|
|
63
|
-
@extractor = extractor
|
|
64
49
|
@opts = opts
|
|
65
50
|
@fallback_anchorless = opts.fetch(:fallback_anchorless, false)
|
|
66
|
-
@link_heuristics = LinkHeuristics.new(url)
|
|
67
|
-
@ignored_cache = {}.compare_by_identity
|
|
68
51
|
end
|
|
69
52
|
|
|
70
|
-
attr_reader :parsed_body
|
|
71
|
-
|
|
72
53
|
##
|
|
73
|
-
# @yieldparam [
|
|
74
|
-
# @return [Enumerator]
|
|
54
|
+
# @yieldparam article [Html2rss::Article]
|
|
55
|
+
# @return [Enumerator]
|
|
75
56
|
def each
|
|
76
57
|
return enum_for(:each) unless block_given?
|
|
77
58
|
|
|
@@ -79,135 +60,77 @@ module Html2rss
|
|
|
79
60
|
end
|
|
80
61
|
|
|
81
62
|
##
|
|
82
|
-
# @return [Boolean]
|
|
63
|
+
# @return [Boolean]
|
|
83
64
|
def extractable?
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
##
|
|
88
|
-
# Decides whether a traversed node has reached a useful article-like
|
|
89
|
-
# boundary for the generic HTML scraper.
|
|
90
|
-
#
|
|
91
|
-
# The predicate prefers containers that add surrounding link context,
|
|
92
|
-
# which helps the scraper move from a leaf anchor toward a repeated
|
|
93
|
-
# teaser/card wrapper.
|
|
94
|
-
#
|
|
95
|
-
# @param node [Nokogiri::XML::Node] candidate boundary node
|
|
96
|
-
# @return [Boolean] true when the node is a good extraction boundary
|
|
97
|
-
def article_tag_condition?(node)
|
|
98
|
-
# Ignore tags that are below ignored DOM chrome.
|
|
99
|
-
return false if Html2rss::Html::Navigator.ignored_container_path?(node, @ignored_cache)
|
|
100
|
-
return true if %w[body html].include?(node.name)
|
|
101
|
-
return false unless (parent = node.parent)
|
|
102
|
-
|
|
103
|
-
anchor_count(parent) > anchor_count(node)
|
|
65
|
+
ranked_segments.any?
|
|
66
|
+
rescue ArgumentError
|
|
67
|
+
false
|
|
104
68
|
end
|
|
105
69
|
|
|
106
70
|
private
|
|
107
71
|
|
|
108
72
|
def articles
|
|
109
73
|
@articles ||= begin
|
|
110
|
-
|
|
111
|
-
|
|
74
|
+
ranked = list_ranked
|
|
75
|
+
if ranked.empty? && @fallback_anchorless
|
|
76
|
+
materialize(cluster_ranked, fallback_anchorless: true)
|
|
77
|
+
else
|
|
78
|
+
materialize(ranked)
|
|
112
79
|
end
|
|
113
|
-
|
|
114
|
-
extracted += find_anchorless_articles if @fallback_anchorless
|
|
115
|
-
extracted
|
|
116
80
|
end
|
|
81
|
+
rescue ArgumentError
|
|
82
|
+
[]
|
|
117
83
|
end
|
|
118
84
|
|
|
119
|
-
def
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
@extractor.new(node, base_url: @url, selected_anchor: nil, fallback_anchorless: true).call
|
|
85
|
+
def ranked_segments
|
|
86
|
+
@ranked_segments ||= begin
|
|
87
|
+
ranked = list_ranked
|
|
88
|
+
ranked = cluster_ranked if @fallback_anchorless && ranked.empty?
|
|
89
|
+
ranked
|
|
125
90
|
end
|
|
126
91
|
end
|
|
127
92
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def minimum_selector_frequency = @opts[:minimum_selector_frequency] || DEFAULT_MINIMUM_SELECTOR_FREQUENCY
|
|
131
|
-
|
|
132
|
-
##
|
|
133
|
-
# @return [Boolean]
|
|
134
|
-
def use_top_selectors = @opts[:use_top_selectors] || DEFAULT_USE_TOP_SELECTORS
|
|
135
|
-
|
|
136
|
-
##
|
|
137
|
-
# @param node [Nokogiri::XML::Node]
|
|
138
|
-
# @return [Integer]
|
|
139
|
-
def anchor_count(node)
|
|
140
|
-
(@anchor_counts ||= {}.compare_by_identity)[node] ||= node.name == 'a' ? 1 : node.css('a').size
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
##
|
|
144
|
-
# @param node [Nokogiri::XML::Node]
|
|
145
|
-
# @return [Boolean]
|
|
146
|
-
def relevant_anchor?(node)
|
|
147
|
-
destination_facts = @link_heuristics.destination_facts(node)
|
|
148
|
-
return false unless destination_facts
|
|
149
|
-
|
|
150
|
-
!noise_anchor?(node, destination_facts:)
|
|
93
|
+
def list_ranked
|
|
94
|
+
@list_ranked ||= rank_strategy(:list, permit_unanchored: false)
|
|
151
95
|
end
|
|
152
96
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
# @yieldparam article_tag [Nokogiri::XML::Node]
|
|
156
|
-
# @yieldparam selected_anchor [Nokogiri::XML::Node]
|
|
157
|
-
# @return [Enumerator, nil]
|
|
158
|
-
def each_article_tag(&block)
|
|
159
|
-
return enum_for(:each_article_tag) unless block
|
|
160
|
-
|
|
161
|
-
anchor_filter = ->(node) { relevant_anchor?(node) }
|
|
162
|
-
boundary_condition = ->(node) { article_tag_condition?(node) }
|
|
163
|
-
|
|
164
|
-
list_candidates.each_article_tag(anchor_filter:, boundary_condition:, &block)
|
|
97
|
+
def cluster_ranked
|
|
98
|
+
@cluster_ranked ||= rank_strategy(:cluster, permit_unanchored: true)
|
|
165
99
|
end
|
|
166
100
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
destination_facts: @link_heuristics.destination_facts(selected_anchor),
|
|
177
|
-
container: article_tag
|
|
101
|
+
def rank_strategy(strategy, permit_unanchored:)
|
|
102
|
+
segments = Segmenter.call(
|
|
103
|
+
document,
|
|
104
|
+
base_url: @url,
|
|
105
|
+
strategy:,
|
|
106
|
+
permit_unanchored:,
|
|
107
|
+
minimum_selector_frequency:,
|
|
108
|
+
use_top_selectors:,
|
|
109
|
+
link_resolver:
|
|
178
110
|
)
|
|
179
|
-
|
|
180
|
-
@extractor.call(article_tag, base_url: @url, selected_anchor:)
|
|
111
|
+
engine.select_eligible(segments, limit: TOP_K)
|
|
181
112
|
end
|
|
182
113
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
# @return [Boolean]
|
|
187
|
-
def noise_anchor?(anchor, destination_facts:, container: nil)
|
|
188
|
-
(@noise_anchors ||= {}.compare_by_identity)[anchor] ||= begin
|
|
189
|
-
text = Html2rss::Html::Navigator.extract_visible_text(anchor).to_s.strip
|
|
190
|
-
@link_heuristics.noise_anchor?(text:, destination_facts:, anchor:, container:)
|
|
114
|
+
def materialize(ranked, fallback_anchorless: false)
|
|
115
|
+
ranked.filter_map do |entry|
|
|
116
|
+
::Html2rss::Html::SstArticleExtractor.call(entry, base_url: @url, scraper: self.class, fallback_anchorless:)
|
|
191
117
|
end
|
|
192
118
|
end
|
|
193
119
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
# @return [Nokogiri::XML::Node, nil]
|
|
197
|
-
def preferred_anchor_for(article_tag)
|
|
198
|
-
article_tag.css(Html2rss::Html::Navigator::MAIN_ANCHOR_SELECTOR).find { relevant_anchor?(_1) } ||
|
|
199
|
-
Html2rss::Html::Navigator.main_anchor_for(article_tag)
|
|
120
|
+
def engine
|
|
121
|
+
@engine ||= Scoring::Engine.new(link_resolver:)
|
|
200
122
|
end
|
|
201
123
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def list_candidates
|
|
205
|
-
Discovery::ListCandidates.new(
|
|
206
|
-
parsed_body,
|
|
207
|
-
minimum_selector_frequency:,
|
|
208
|
-
use_top_selectors:
|
|
209
|
-
)
|
|
124
|
+
def link_resolver
|
|
125
|
+
@link_resolver ||= @provided_link_resolver || Scoring::LinkResolver.new(@url)
|
|
210
126
|
end
|
|
127
|
+
|
|
128
|
+
def document
|
|
129
|
+
@document ||= @provided_document || SST::Normalizer.call(@parsed_body)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def minimum_selector_frequency = @opts[:minimum_selector_frequency] || DEFAULT_MINIMUM_SELECTOR_FREQUENCY
|
|
133
|
+
def use_top_selectors = @opts[:use_top_selectors] || DEFAULT_USE_TOP_SELECTORS
|
|
211
134
|
end
|
|
212
135
|
end
|
|
213
136
|
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class AutoSource
|
|
5
|
+
module Scraper
|
|
6
|
+
class JsonState
|
|
7
|
+
# Shapes raw entries into the structure required downstream.
|
|
8
|
+
module ArticleNormalizer
|
|
9
|
+
# Preferred keys when extracting title-like values from state payloads.
|
|
10
|
+
TITLE_KEYS = %i[title headline name text].freeze
|
|
11
|
+
# Preferred keys when extracting URL-like values from state payloads.
|
|
12
|
+
URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
|
|
13
|
+
# Preferred keys when extracting description-like values from state payloads.
|
|
14
|
+
DESCRIPTION_KEYS = %i[description summary excerpt dek subheading].freeze
|
|
15
|
+
# Preferred keys when extracting image-like values from state payloads.
|
|
16
|
+
IMAGE_KEYS = %i[image imageUrl thumbnailUrl thumbnail src featuredImage coverImage heroImage].freeze
|
|
17
|
+
# Preferred keys when extracting publication timestamps from state payloads.
|
|
18
|
+
PUBLISHED_AT_KEYS = %i[published_at publishedAt datePublished date publicationDate pubDate updatedAt
|
|
19
|
+
updated_at createdAt created_at].freeze
|
|
20
|
+
# Preferred keys when extracting category-like values from state payloads.
|
|
21
|
+
CATEGORY_KEYS = %i[categories tags section sections topic topics channel].freeze
|
|
22
|
+
# Preferred keys when extracting identifier-like values from state payloads.
|
|
23
|
+
ID_KEYS = %i[id guid uuid slug key].freeze
|
|
24
|
+
|
|
25
|
+
module_function
|
|
26
|
+
|
|
27
|
+
# rubocop:disable Metrics/MethodLength
|
|
28
|
+
# @param entry [Hash] raw article entry candidate
|
|
29
|
+
# @param base_url [String, Html2rss::Url] base URL for relative link resolution
|
|
30
|
+
# @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
|
|
31
|
+
def normalise(entry, base_url:)
|
|
32
|
+
return unless entry.is_a?(Hash)
|
|
33
|
+
|
|
34
|
+
title = string(ValueFinder.fetch(entry, TITLE_KEYS))
|
|
35
|
+
description = string(ValueFinder.fetch(entry, DESCRIPTION_KEYS))
|
|
36
|
+
article_url = resolve_link(entry, keys: URL_KEYS, base_url:,
|
|
37
|
+
log_key: 'JsonState: invalid URL encountered')
|
|
38
|
+
return unless article_url
|
|
39
|
+
return if title.nil? && description.nil?
|
|
40
|
+
|
|
41
|
+
{
|
|
42
|
+
title:,
|
|
43
|
+
description:,
|
|
44
|
+
url: article_url,
|
|
45
|
+
image: resolve_link(entry, keys: IMAGE_KEYS, base_url:,
|
|
46
|
+
log_key: 'JsonState: invalid image URL encountered'),
|
|
47
|
+
published_at: string(ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)),
|
|
48
|
+
categories: categories(entry),
|
|
49
|
+
id: identifier(entry, article_url)
|
|
50
|
+
}.compact
|
|
51
|
+
end
|
|
52
|
+
# rubocop:enable Metrics/MethodLength
|
|
53
|
+
|
|
54
|
+
# @param value [Object] candidate scalar value
|
|
55
|
+
# @return [String, nil] normalized non-empty string value
|
|
56
|
+
def string(value)
|
|
57
|
+
trimmed = value.to_s.strip
|
|
58
|
+
trimmed unless trimmed.empty?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @param entry [Hash] raw article entry candidate
|
|
62
|
+
# @param keys [Array<String>] preferred link keys
|
|
63
|
+
# @param base_url [String, Html2rss::Url] base URL for relative link resolution
|
|
64
|
+
# @param log_key [String] structured log message key
|
|
65
|
+
# @return [Html2rss::Url, nil] resolved absolute URL
|
|
66
|
+
def resolve_link(entry, keys:, base_url:, log_key:)
|
|
67
|
+
value = ValueFinder.fetch(entry, keys)
|
|
68
|
+
value = ValueFinder.fetch(value, keys) if value.is_a?(Hash)
|
|
69
|
+
string = string(value)
|
|
70
|
+
return unless string
|
|
71
|
+
|
|
72
|
+
Url.from_relative(string, base_url)
|
|
73
|
+
rescue ArgumentError
|
|
74
|
+
Log.debug(log_key, url: string)
|
|
75
|
+
nil
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# rubocop:disable Metrics/MethodLength
|
|
79
|
+
# @param entry [Hash] raw article entry candidate
|
|
80
|
+
# @return [Array<String>, nil] normalized unique categories
|
|
81
|
+
def categories(entry)
|
|
82
|
+
raw = ValueFinder.fetch(entry, CATEGORY_KEYS)
|
|
83
|
+
names = case raw
|
|
84
|
+
when Array then raw
|
|
85
|
+
when Hash then raw.values
|
|
86
|
+
when String then [raw]
|
|
87
|
+
else []
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
result = names.flat_map do |value|
|
|
91
|
+
case value
|
|
92
|
+
when Hash
|
|
93
|
+
string(ValueFinder.fetch(value, %i[name title label]))
|
|
94
|
+
else
|
|
95
|
+
string(value)
|
|
96
|
+
end
|
|
97
|
+
end.compact
|
|
98
|
+
|
|
99
|
+
result.uniq!
|
|
100
|
+
result unless result.empty?
|
|
101
|
+
end
|
|
102
|
+
# rubocop:enable Metrics/MethodLength
|
|
103
|
+
|
|
104
|
+
# @param entry [Hash] raw article entry candidate
|
|
105
|
+
# @param article_url [Html2rss::Url] resolved article URL
|
|
106
|
+
# @return [String] stable article identifier fallbacking to resolved URL
|
|
107
|
+
def identifier(entry, article_url)
|
|
108
|
+
value = ValueFinder.fetch(entry, ID_KEYS)
|
|
109
|
+
value = ValueFinder.fetch(value, ID_KEYS) if value.is_a?(Hash)
|
|
110
|
+
string(value) || article_url.to_s
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class AutoSource
|
|
5
|
+
module Scraper
|
|
6
|
+
class JsonState
|
|
7
|
+
# Identifies arrays that look like collections of article hashes.
|
|
8
|
+
module CandidateDetector
|
|
9
|
+
# Preferred keys when extracting title-like values from state payloads.
|
|
10
|
+
TITLE_KEYS = %i[title headline name text].freeze
|
|
11
|
+
# Preferred keys when extracting URL-like values from state payloads.
|
|
12
|
+
URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl].freeze
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# @param document [Hash, Array, Object] candidate document node
|
|
17
|
+
# @return [Boolean] whether the node contains article-like arrays
|
|
18
|
+
def candidate_array?(document)
|
|
19
|
+
case document
|
|
20
|
+
when Array
|
|
21
|
+
return true if array_of_articles?(document)
|
|
22
|
+
|
|
23
|
+
document.any? { traversable_candidate?(_1) }
|
|
24
|
+
when Hash then document.each_value.any? { candidate_array?(_1) }
|
|
25
|
+
else false
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# @param value [Hash, Array, Object] candidate nested value
|
|
30
|
+
# @return [Boolean] whether nested value should be traversed for article candidates
|
|
31
|
+
def traversable_candidate?(value)
|
|
32
|
+
case value
|
|
33
|
+
when Array, Hash then candidate_array?(value)
|
|
34
|
+
else false
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param array [Array<Object>] candidate list of entries
|
|
39
|
+
# @return [Boolean] whether array includes hash entries with title and URL fields
|
|
40
|
+
def array_of_articles?(array)
|
|
41
|
+
array.any? do |element|
|
|
42
|
+
next unless element.is_a?(Hash)
|
|
43
|
+
|
|
44
|
+
title_from(element) && url_from(element)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @param object [Hash] article candidate object
|
|
49
|
+
# @return [Object, nil] detected title-like value
|
|
50
|
+
def title_from(object)
|
|
51
|
+
ValueFinder.fetch(object, TITLE_KEYS)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# @param object [Hash] article candidate object
|
|
55
|
+
# @return [Object, nil] detected URL-like value
|
|
56
|
+
def url_from(object)
|
|
57
|
+
ValueFinder.fetch(object, URL_KEYS)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|