html2rss 0.34.1 → 0.35.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +14 -0
  3. data/lib/html2rss/article/enrichment.rb +179 -0
  4. data/lib/html2rss/article.rb +12 -0
  5. data/lib/html2rss/auto_source/scraper/sitemap.rb +1 -1
  6. data/lib/html2rss/capture.rb +2 -1
  7. data/lib/html2rss/config/validator.rb +7 -0
  8. data/lib/html2rss/config.rb +2 -0
  9. data/lib/html2rss/feed_pipeline.rb +49 -10
  10. data/lib/html2rss/html/article_extractor/image_extractor.rb +1 -1
  11. data/lib/html2rss/html/article_extractor.rb +1 -1
  12. data/lib/html2rss/html/article_rules/date.rb +11 -0
  13. data/lib/html2rss/html/article_rules/url_date.rb +78 -0
  14. data/lib/html2rss/html/sst_article_extractor.rb +2 -2
  15. data/lib/html2rss/link_destination/destination_facts.rb +1 -9
  16. data/lib/html2rss/link_destination/path_classifier.rb +31 -5
  17. data/lib/html2rss/request_service/httpx_strategy.rb +1 -1
  18. data/lib/html2rss/request_service/network_guard.rb +35 -6
  19. data/lib/html2rss/request_service/network_guard_filter.rb +84 -0
  20. data/lib/html2rss/request_session/pager/custom_selector.rb +1 -3
  21. data/lib/html2rss/request_session/pager/json_cursor.rb +1 -1
  22. data/lib/html2rss/request_session/pager/rel_next.rb +1 -3
  23. data/lib/html2rss/request_session.rb +20 -0
  24. data/lib/html2rss/selectors/extractors/href.rb +2 -2
  25. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +2 -1
  26. data/lib/html2rss/selectors/post_processors/parse_uri.rb +2 -2
  27. data/lib/html2rss/selectors.rb +1 -4
  28. data/lib/html2rss/status.rb +26 -9
  29. data/lib/html2rss/url.rb +54 -3
  30. data/lib/html2rss/version.rb +1 -1
  31. data/schema/html2rss-config.schema.json +20 -0
  32. metadata +5 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e91a63f8267b667a0bc1fc7ea2a17419b4386b7afa30d081c1bda9dc6f5ccd5f
4
- data.tar.gz: fb785c14e1e6463d00f77b0cbc3ecf16689e3f326f5cda333e183f8c4db5c61c
3
+ metadata.gz: db36711f78369ba0783ffe4203df047b431224926ce3ea8528985f0498237653
4
+ data.tar.gz: c819bec563f9ec056d909861b6c62ef85afcc7411906404c88e731248c3b17a5
5
5
  SHA512:
6
- metadata.gz: 1cfd939a70acd9a4ce61c7b2ce4f06e3fbac729baec3e34c795c5b6d2f042d40f195932c24bea776b83da98f12b8020d4cb383814d42be6ba45c2ba133611773
7
- data.tar.gz: c3b43b4bfffa6ad0e1873a76d0aef74935b4512a4d8262eb4b156207f634f39942c0df265776a35a1d3e8218f1c002a336fb6d16128fbfdebb9139a216cf11c5
6
+ metadata.gz: 54dedcfaea4d2d521d6dcc9a8850513f9e946c10c3f951c569ba1bc21662791be5d727bac5d27d654bb7c3941378e51887bdc05f3aa301b738dd12bc17b946ae
7
+ data.tar.gz: a9c66a90a7ce89be190ed0179c72833bdbbf0c9ef014e27dfb639cef76de35d47a95d29713ce59ef79f3370dabad7b1a5a4585f8015138a319d476a28fc63b06
data/README.md CHANGED
@@ -184,6 +184,20 @@ docker compose -f docker-compose.botasaurus.yml up -d
184
184
 
185
185
  Set `BOTASAURUS_SCRAPER_URL` to `http://127.0.0.1:4010` and use strategy `botasaurus` in MCP tools, Capture, or the CLI.
186
186
 
187
+ ## Article-page enrichment
188
+
189
+ Listing pages often lack a date, author or image. With `enrich` set, the feed build fetches the item's own page and fills only those missing fields from its JSON-LD (the page's own `author` and `datePublished`) and meta tags (`og:image`, `article:published_time`, `article:author`). Values already found on the listing are never overwritten, and an article-page date wins over a date guessed from the URL. Titles, descriptions and body text are never touched.
190
+
191
+ ```yaml
192
+ enrich:
193
+ article_metadata: true # default: off
194
+ max_fetches: 10 # default and hard maximum: 10
195
+ ```
196
+
197
+ Cost: up to `max_fetches` extra requests per build, on top of the normal request budget, using the same request strategy. Links must stay on the feed's origin; others are skipped. A failed or blocked fetch leaves the item unchanged and never fails the build.
198
+
199
+ `--explain` prints three counts when enrichment ran: `enriched` (items that gained a field), `enrich_failed` (failed or blocked fetches, including cross-origin skips) and `enrich_skipped_cross_origin` (the cross-origin subset).
200
+
187
201
  ## Request Strategies
188
202
 
189
203
  | Strategy | Description |
@@ -0,0 +1,179 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cgi'
4
+
5
+ module Html2rss
6
+ class Article
7
+ ##
8
+ # Opt-in feed-build step and the single owner of "metadata from an article page": for items missing
9
+ # +published_at+, +author+ or +image+ it fetches the item's own page and fills only those fields from the
10
+ # page's JSON-LD and meta tags. Listing values are never overwritten; body text and description are never read.
11
+ #
12
+ # Fetches go through a {RequestSession} sub-allowance, so strategy, policy (same-origin rule, network guard,
13
+ # response size cap) and headers match the listing fetch. The allowance is separate from the listing
14
+ # budget, so one build makes at most {MAX_TOTAL_REQUESTS} requests: the listing budget
15
+ # ({RequestService::Policy::MAX_REQUESTS_CEILING}) plus {MAX_FETCHES}. A blocked or failed fetch leaves the
16
+ # item unchanged and is counted; enrichment never fails the build.
17
+ class Enrichment # rubocop:disable Metrics/ClassLength -- fetch, parse and fill share one owner
18
+ # Hard maximum and default number of article fetches per feed build.
19
+ MAX_FETCHES = 10
20
+ # Upper bound on requests one build can make: listing budget plus the enrichment allowance.
21
+ MAX_TOTAL_REQUESTS = RequestService::Policy::MAX_REQUESTS_CEILING + MAX_FETCHES
22
+ # Per field an article page can supply: how to read it from an article and how to set it.
23
+ FIELDS = {
24
+ published_at: [lambda(&:published_at), ->(article, value) { article.with_published_at(value) }],
25
+ author: [lambda(&:author), ->(article, value) { article.with_author(value) }],
26
+ image: [lambda(&:image), ->(article, value) { article.with_image(value) }]
27
+ }.freeze
28
+ # Schema.org keys that are weaker sources than a Thing's own author and datePublished.
29
+ WEAK_KEYS = %i[publisher hiringOrganization organizer datePosted dateCreated dateModified uploadDate].freeze
30
+ # Schema.org types that are the page's article rather than a related item.
31
+ ARTICLE_TYPES = Set['Article', 'NewsArticle', 'BlogPosting'].freeze
32
+ # Meta tag keys per field, in precedence order.
33
+ META_KEYS = {
34
+ published_at: %w[article:published_time],
35
+ author: %w[article:author author],
36
+ image: %w[og:image]
37
+ }.freeze
38
+ # Result of one run. +tally+ keys: enriched (items that gained a field), enrich_failed (fetches that
39
+ # failed or were blocked, including cross-origin skips), enrich_skipped_cross_origin (the policy-denied subset).
40
+ Result = Data.define(:articles, :tally)
41
+
42
+ class << self
43
+ ##
44
+ # @param articles [Array<Html2rss::Article>] deduplicated articles
45
+ # @param request_session [Html2rss::RequestSession] the listing session (strategy and policy are reused)
46
+ # @param max_fetches [Integer] article fetch cap, capped at {MAX_FETCHES}
47
+ # @return [Result] the articles (unchanged ones kept in place) and the enrichment tally
48
+ def call(articles, request_session:, max_fetches: MAX_FETCHES)
49
+ cap = [max_fetches, MAX_FETCHES].min
50
+ new(request_session.with_budget_allowance(cap), cap).call(articles)
51
+ end
52
+ end
53
+
54
+ # @param session [Html2rss::RequestSession] session with the enrichment request allowance
55
+ # @param cap [Integer] maximum article fetches
56
+ def initialize(session, cap)
57
+ @session = session
58
+ @cap = cap
59
+ @tally = {}
60
+ end
61
+
62
+ # @param articles [Array<Html2rss::Article>]
63
+ # @return [Result]
64
+ def call(articles)
65
+ budget = @cap
66
+ enriched = articles.map do |article|
67
+ next article unless budget.positive? && candidate?(article)
68
+
69
+ budget -= 1
70
+ enrich(article)
71
+ end
72
+ Result.new(articles: enriched, tally: @tally)
73
+ end
74
+
75
+ private
76
+
77
+ def candidate?(article)
78
+ article.url && FIELDS.each_value.any? { |read, _| read.call(article).nil? }
79
+ end
80
+
81
+ def enrich(article)
82
+ response = fetch(article.url)
83
+ return failed(article) unless page?(response)
84
+
85
+ fill(article, page_metadata(response))
86
+ rescue RequestService::CrossOriginFollowUpDenied
87
+ count(:enrich_skipped_cross_origin)
88
+ failed(article)
89
+ rescue Html2rss::ContractError
90
+ raise
91
+ rescue Html2rss::Error, JSON::ParserError
92
+ failed(article)
93
+ end
94
+
95
+ # Sets only the fields the article lacks; counts the article when any was set.
96
+ def fill(article, found)
97
+ filled = FIELDS.reduce(article) do |current, (field, (read, write))|
98
+ read.call(current).nil? && found[field] ? write.call(current, found.fetch(field)) : current
99
+ end
100
+ count(:enriched) unless filled.equal?(article)
101
+ filled
102
+ end
103
+
104
+ def count(key)
105
+ @tally[key] = @tally.fetch(key, 0) + 1
106
+ end
107
+
108
+ def failed(article)
109
+ count(:enrich_failed)
110
+ article
111
+ end
112
+
113
+ def fetch(url)
114
+ @session.follow_up(url:, relation: :auto_source, origin_url: @session.url)
115
+ end
116
+
117
+ def page?(response)
118
+ (response.status.nil? || response.status.between?(200, 299)) && response.html_response?
119
+ end
120
+
121
+ def page_metadata(response)
122
+ document = response.parsed_body
123
+ things = jsonld_things(document, response.url)
124
+ FIELDS.keys.to_h { |field| [field, jsonld_value(things, field) || meta_value(document, field, response.url)] }
125
+ end
126
+
127
+ # The page's own article Things, best first: the Thing whose url matches the page, then article-typed
128
+ # ones, then document order. Each Thing is built without the weaker sources {WEAK_KEYS} (Thing#author
129
+ # would fall back to publisher; Thing#published_at to modified/upload dates).
130
+ def jsonld_things(document, url)
131
+ things = document.css('script[type="application/ld+json"]').flat_map { thing_objects(_1) }
132
+ .map { AutoSource::Scraper::Schema::Thing.new(_1.except(*WEAK_KEYS), url:) }
133
+ things.group_by { thing_rank(_1, url) }.sort.flat_map(&:last)
134
+ end
135
+
136
+ def thing_objects(script)
137
+ schema = AutoSource::Scraper::Schema
138
+ schema.from_script(script).select { schema.scraper_for_schema_object(_1) == schema::Thing }
139
+ rescue JSON::ParserError
140
+ []
141
+ end
142
+
143
+ def thing_rank(thing, page_url)
144
+ return 0 if comparable(thing.url) == comparable(page_url)
145
+
146
+ types = AutoSource::Scraper::Schema.normalize_types(thing.schema_object[:@type])
147
+ types.intersect?(ARTICLE_TYPES) ? 1 : 2
148
+ end
149
+
150
+ def comparable(url) = url.to_s.chomp('/')
151
+
152
+ def jsonld_value(things, field)
153
+ things.each do |thing|
154
+ value = thing_value(thing, field).to_s.strip
155
+ return value unless value.empty?
156
+ end
157
+ nil
158
+ end
159
+
160
+ def thing_value(thing, field)
161
+ case field
162
+ when :published_at then thing.published_at
163
+ when :author then thing.author
164
+ else thing.image
165
+ end
166
+ end
167
+
168
+ def meta_value(document, field, base)
169
+ META_KEYS.fetch(field).each do |key|
170
+ value = document.at_css("meta[property='#{key}'], meta[name='#{key}']")&.[]('content').to_s.strip
171
+ next if value.empty? || (field == :author && value.include?('://'))
172
+
173
+ return field == :image ? Url.from_relative(value, base).to_s : value
174
+ end
175
+ nil
176
+ end
177
+ end
178
+ end
179
+ end
@@ -60,6 +60,18 @@ module Html2rss
60
60
  linked? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
61
61
  end
62
62
 
63
+ # @param value [DateTime, String, nil] replacement publication timestamp
64
+ # @return [Article] a copy of this article carrying +value+ as published_at
65
+ def with_published_at(value) = self.class.new(**@to_h, published_at: value)
66
+
67
+ # @param value [String, nil] replacement author name
68
+ # @return [Article] a copy of this article carrying +value+ as author
69
+ def with_author(value) = self.class.new(**@to_h, author: value)
70
+
71
+ # @param value [String, Html2rss::Url, nil] replacement image URL
72
+ # @return [Article] a copy of this article carrying +value+ as image
73
+ def with_image(value) = self.class.new(**@to_h, image: value)
74
+
63
75
  # @param new_url [String, Html2rss::Url] replacement article URL
64
76
  # @return [Article] a copy of this article carrying +new_url+
65
77
  def with_url(new_url) = self.class.new(**@to_h, url: new_url)
@@ -151,7 +151,7 @@ module Html2rss
151
151
  def target_sitemap_url
152
152
  link_tag = parsed_body.at_css(SITEMAP_LINK_SELECTOR)
153
153
  href = link_tag ? link_tag['href'] : '/sitemap.xml'
154
- Html2rss::Url.from_relative(href, url)
154
+ Html2rss::Url.resolve_web(href, url) || Html2rss::Url.resolve_web('/sitemap.xml', url)
155
155
  end
156
156
 
157
157
  def article_from_entry(entry)
@@ -329,7 +329,8 @@ module Html2rss
329
329
  end
330
330
 
331
331
  def find_matching_article(segment, articles_by_url)
332
- resolved = Url.from_relative(segment.primary_link.attrs.href.to_s, @url)
332
+ return unless (resolved = Url.resolve_web(segment.primary_link.attrs.href, @url, web_only: false))
333
+
333
334
  articles_by_url[resolved.to_s] || articles_by_url.values.find { |a| fuzzy_url_match?(a.url, resolved) }
334
335
  end
335
336
 
@@ -93,6 +93,12 @@ module Html2rss
93
93
  optional(:local_file_path).filled(:string)
94
94
  end
95
95
 
96
+ # Contract for the opt-in `enrich` section (article-page metadata for items missing date, author or image).
97
+ EnrichConfig = Dry::Schema.Params do
98
+ optional(:article_metadata).filled(:bool)
99
+ optional(:max_fetches).filled(:integer, gt?: 0, lteq?: Html2rss::Article::Enrichment::MAX_FETCHES)
100
+ end
101
+
96
102
  params do
97
103
  optional(:strategy).filled(:symbol)
98
104
  required(:channel).hash(ChannelConfig)
@@ -103,6 +109,7 @@ module Html2rss
103
109
  optional(:selectors).hash
104
110
  optional(:dynamic_params_error).maybe(:string)
105
111
  optional(:request).hash(RequestConfig)
112
+ optional(:enrich).hash(EnrichConfig)
106
113
  end
107
114
 
108
115
  rule(:headers) do
@@ -316,6 +316,8 @@ module Html2rss
316
316
 
317
317
  # @return [Hash{Symbol => Object, nil}] selectors configuration
318
318
  def selectors = config[:selectors]
319
+ # @return [Hash{Symbol => Object}, nil] article-page enrichment options (+article_metadata+, +max_fetches+)
320
+ def enrich = config[:enrich]
319
321
  # @return [Hash{Symbol => Object, nil}] auto-source configuration
320
322
  def auto_source = config[:auto_source]
321
323
 
@@ -6,22 +6,37 @@ module Html2rss
6
6
  #
7
7
  # {include:file:lib/html2rss/feed_pipeline/README.md}
8
8
  class FeedPipeline # rubocop:disable Metrics/ClassLength -- outcome + collect seams stay co-located
9
+ PipelineOutcome = Data.define(
10
+ :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
11
+ :admission_drops, :scrape_target, :entry_resolution, :feed_channel, :enrichment
12
+ )
13
+
14
+ Extraction = Data.define(:articles, :dedup_dropped, :admission_drops, :feed_channel, :enrichment)
15
+
9
16
  # Scrape-finished facts after request + extraction + dedup (before Channel/Status materialize).
10
17
  # selected_strategy: set on :auto success; nil otherwise.
11
18
  # attempt_count: auto attempts attempted; 0 outside :auto.
12
19
  # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
13
20
  # feed_channel: channel facts of the native feed followed ({Syndication::ChannelFacts}); nil without one.
14
- PipelineOutcome = Data.define(
15
- :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
16
- :admission_drops, :scrape_target, :entry_resolution, :feed_channel
17
- )
21
+ # enrichment: {Article::Enrichment} tally (empty when enrichment is off).
22
+ class PipelineOutcome
23
+ # @param enrichment [Hash{Symbol => Integer}] defaults to an empty tally (enrichment off)
24
+ # @param fields [Hash{Symbol => Object}] the remaining members
25
+ # @option fields [Object] :articles the other members, as named above
26
+ def initialize(enrichment: {}, **fields) = super
27
+ end
18
28
 
19
29
  # What one response yields after extraction and dedup ({#extract}); its fields are the
20
30
  # {PipelineOutcome} fields of the same names.
21
31
  # articles: unique articles. dedup_dropped: articles removed by deduplication.
22
32
  # admission_drops: Cleanup drop tallies. feed_channel: the followed native feed's channel facts
23
- # ({Syndication::ChannelFacts}); nil without one.
24
- Extraction = Data.define(:articles, :dedup_dropped, :admission_drops, :feed_channel)
33
+ # ({Syndication::ChannelFacts}); nil without one. enrichment: {Article::Enrichment} tally (empty when off).
34
+ class Extraction
35
+ # @param enrichment [Hash{Symbol => Integer}] defaults to an empty tally (enrichment off)
36
+ # @param fields [Hash{Symbol => Object}] the remaining members
37
+ # @option fields [Object] :articles the other members, as named above
38
+ def initialize(enrichment: {}, **fields) = super
39
+ end
25
40
 
26
41
  ##
27
42
  # @param raw_config [Hash{Symbol => Object}] user-provided feed config
@@ -82,14 +97,38 @@ module Html2rss
82
97
  # @return [Extraction] unique articles with their dedup and admission drops and feed channel facts
83
98
  def extract(config:, response:, request_session:)
84
99
  collected, admission_drops, feed_channel = collect_articles(config:, response:, request_session:)
85
- unique = Article::Deduplicator.new(collected).call
86
- Extraction.new(articles: unique, dedup_dropped: collected.size - unique.size, admission_drops:, feed_channel:)
100
+ deduped = Article::Deduplicator.new(collected).call
101
+ enriched, enrichment = enrich(deduped, config:, request_session:)
102
+ unique = with_url_dates(enriched, config.time_zone)
103
+ Extraction.new(articles: unique, dedup_dropped: collected.size - deduped.size, admission_drops:, feed_channel:,
104
+ enrichment:)
87
105
  end
88
106
 
89
107
  private
90
108
 
91
109
  attr_reader :raw_config
92
110
 
111
+ # Opt-in article-page metadata. Runs before {#with_url_dates} so a real article-page date wins over a URL date.
112
+ def enrich(articles, config:, request_session:)
113
+ return [articles, {}] unless (options = config.enrich)&.fetch(:article_metadata, false)
114
+
115
+ result = Article::Enrichment.call(
116
+ articles, request_session:, max_fetches: options.fetch(:max_fetches, Article::Enrichment::MAX_FETCHES)
117
+ )
118
+ [result.articles, result.tally]
119
+ end
120
+
121
+ # The single owner of the URL-path date fallback: it runs on the final articles, after cross-scraper
122
+ # merge and dedup, so no earlier stage (card walk, merge, scoring) ever sees a URL date.
123
+ def with_url_dates(articles, time_zone)
124
+ articles.map do |article|
125
+ next article if article.published_at
126
+
127
+ date = Html::ArticleRules::UrlDate.call(article.url, time_zone:)
128
+ date ? article.with_published_at(date) : article
129
+ end
130
+ end
131
+
93
132
  def pipeline_outcome_for(config)
94
133
  plan = StrategyPlan.resolve(config.strategy)
95
134
  resources = RuntimePolicy.resources_for(config)
@@ -181,7 +220,7 @@ module Html2rss
181
220
  [articles, {}, feed.channel]
182
221
  end
183
222
 
184
- def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
223
+ def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
185
224
  channel = Channel.from_response(outcome.response, overrides: config.channel, feed: outcome.feed_channel)
186
225
  status = Status.build(
187
226
  articles: outcome.articles,
@@ -191,7 +230,7 @@ module Html2rss
191
230
  strategy_attempts: outcome.strategy_attempts,
192
231
  admission_drops: outcome.admission_drops,
193
232
  scrape_target: outcome.scrape_target,
194
- entry_resolution: outcome.entry_resolution
233
+ entry_resolution: outcome.entry_resolution, enrichment: outcome.enrichment
195
234
  )
196
235
  FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
197
236
  end
@@ -15,7 +15,7 @@ module Html2rss
15
15
  from_style(article_tag) ||
16
16
  from_poster(article_tag)
17
17
 
18
- Url.from_relative(img_src, base_url) if img_src
18
+ Url.resolve_web(img_src, base_url)
19
19
  end
20
20
 
21
21
  # @param article_tag [Nokogiri::XML::Element] article container node
@@ -62,7 +62,7 @@ module Html2rss
62
62
  def extract_url
63
63
  @extract_url ||= begin
64
64
  href = selected_anchor&.[]('href').to_s
65
- Url.from_relative(href.split('#').first, base_url) unless href.empty?
65
+ Url.resolve_web(href.split('#').first, base_url, web_only: false)
66
66
  end
67
67
  end
68
68
 
@@ -31,6 +31,17 @@ module Html2rss
31
31
  values.min
32
32
  end
33
33
 
34
+ ##
35
+ # @param time_zone [String] channel time zone
36
+ # @return [::Date] today's calendar day in +time_zone+
37
+ def local_today(time_zone:) = timezone_for(time_zone).now.to_date
38
+
39
+ ##
40
+ # @param date [::Date] calendar day
41
+ # @param time_zone [String] channel time zone
42
+ # @return [DateTime] midnight of +date+ in +time_zone+
43
+ def at_midnight(date, time_zone:) = localize(DateTime.new(date.year, date.month, date.day), time_zone)
44
+
34
45
  private
35
46
 
36
47
  def parse_attr(value, time_zone:)
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'date'
4
+
5
+ module Html2rss
6
+ module Html
7
+ module ArticleRules
8
+ ##
9
+ # Sole owner of "publication date from an article URL path". It is applied once, on the final
10
+ # article, only when no other date source filled +published_at+. Precision over recall.
11
+ #
12
+ # Algorithm adapted from newspaper4k's DATE_REGEX (MIT), tightened to two shapes, each year-first,
13
+ # starting a path segment and followed by a non-empty slug: +/YYYY/M/D/slug+ (day is its own segment)
14
+ # and +YYYY-MM-DD-slug+ or +YYYY_MM_DD/slug+ (one in-segment separator, two-digit month and day).
15
+ # Numeric ids (+58201224+), day-less paths (+/2020/06/88083+), bare +/2024/01/01/+,
16
+ # +sku-2023-10-15-blue+, WordPress +/2024/05/10-best-laptops+ and CalVer +/docs/2024.1.2-notes+
17
+ # therefore yield nothing. The result is day precision: midnight in the channel time zone.
18
+ #
19
+ # Accepted residual: evergreen or past-event pages whose path carries a date get that date, since
20
+ # a path cannot tell publication from event date.
21
+ module UrlDate
22
+ # Calendar-year rule (bounded, 1995..today), unlike PathClassifier::YEARISH_SEGMENT's routing shape.
23
+ # Earliest year accepted as a publication year.
24
+ MIN_YEAR = 1995
25
+ # A whole segment that is a plausible-looking four-digit year.
26
+ YEAR = /\A(?:19|20)\d{2}\z/
27
+ # A whole segment of one or two digits (month or day).
28
+ DAY = /\A\d{1,2}\z/
29
+ # One segment +YYYY-MM-DD+ or +YYYY_MM_DD+ (same separator), then +-+/+_+ and a slug, or its end
30
+ # (a later segment must then be the slug).
31
+ JOINED = /\A((?:19|20)\d{2})([-_])(\d{2})\2(\d{2})(?:[-_].|(\z))/m
32
+
33
+ class << self
34
+ ##
35
+ # @param url [Html2rss::Url, String, nil] article URL (only its path is read)
36
+ # @param time_zone [String] channel time zone
37
+ # @return [DateTime, nil] local midnight of the one unambiguous path date
38
+ def call(url, time_zone:)
39
+ dates = path_dates(Url.from_absolute(url).decoded_path_segments, time_zone:)
40
+ return unless dates.one?
41
+
42
+ Date.at_midnight(dates.first, time_zone:)
43
+ rescue Url::InvalidUrlError
44
+ nil
45
+ end
46
+
47
+ private
48
+
49
+ def path_dates(segments, time_zone:)
50
+ latest = Date.local_today(time_zone:) + 1
51
+ candidates = segments.each_index.filter_map { candidate(segments.drop(_1)) }
52
+ candidates.map { plausible(_1.map(&:to_i), latest) }.uniq
53
+ end
54
+
55
+ # +/YYYY/M/D/slug+ is four consecutive segments; the joined shape is one segment, so a decoded
56
+ # +%2F+ inside a segment never makes a boundary.
57
+ def candidate(segments)
58
+ case segments
59
+ in [YEAR => year, DAY => month, DAY => day, _, *] then [year, month, day]
60
+ in [segment, *rest] if (match = JOINED.match(segment)) && (!match[5] || rest.any?)
61
+ match.values_at(1, 3, 4)
62
+ else
63
+ nil
64
+ end
65
+ end
66
+
67
+ # A path with any implausible or second distinct date is ambiguous, so nil stays in the list.
68
+ def plausible(parts, latest)
69
+ return unless ::Date.valid_date?(*parts)
70
+
71
+ date = ::Date.new(*parts)
72
+ date if date.year >= MIN_YEAR && date <= latest
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -72,7 +72,7 @@ module Html2rss
72
72
  def extract_url
73
73
  @extract_url ||= begin
74
74
  href = @selected_anchor.attrs.href.to_s
75
- Url.from_relative(href.split('#').first.strip, @base_url) unless href.empty?
75
+ Url.resolve_web(href.split('#').first, @base_url, web_only: false)
76
76
  end
77
77
  end
78
78
 
@@ -211,7 +211,7 @@ module Html2rss
211
211
 
212
212
  def extract_image
213
213
  img_src = image_from_srcset || image_from_img || image_from_style || image_from_poster
214
- Url.from_relative(img_src, @base_url) if img_src
214
+ Url.resolve_web(img_src, @base_url)
215
215
  end
216
216
 
217
217
  def image_from_img
@@ -41,18 +41,10 @@ module Html2rss
41
41
  # @param url [Html2rss::Url]
42
42
  # @return [PathClassifier] path classifier that also sees the first host label (an obituary subdomain)
43
43
  def self.classifier_for(url)
44
- PathClassifier.new(url.path_segments.map { decode_segment(_1) },
44
+ PathClassifier.new(url.decoded_path_segments,
45
45
  host_label: url.host.to_s.split('.').first, query: !url.query.nil?)
46
46
  end
47
47
  private_class_method :classifier_for
48
-
49
- # Url keeps path segments percent-encoded; classification words are plain NFC text.
50
- # @param segment [String] percent-encoded path segment
51
- # @return [String] decoded, NFC-normalized segment
52
- def self.decode_segment(segment)
53
- Addressable::URI.unencode_component(segment).force_encoding(Encoding::UTF_8).scrub.unicode_normalize(:nfc)
54
- end
55
- private_class_method :decode_segment
56
48
  end
57
49
  end
58
50
  end
@@ -105,7 +105,7 @@ module Html2rss
105
105
  deep_post_context: Set['press', 'newsroom', 'presse', 'pressemitteilungen', 'prensa'].freeze
106
106
  }.freeze
107
107
  end
108
- # Path segment that begins with a year-like publishing marker.
108
+ # Routing shape: any digit-led slug or id (4+ digits), not a calendar year; see ArticleRules::UrlDate::YEAR.
109
109
  YEARISH_SEGMENT = /\A\d{4,}[\w-]*\z/
110
110
  # Hyphenated slug shape common to article permalinks.
111
111
  POST_SLUG_SEGMENT = /\A[a-z0-9]+(?:-[a-z0-9]+){2,}\z/i
@@ -113,7 +113,18 @@ module Html2rss
113
113
  # Final label must be alphabetic (TLD-like); exclude common file extensions.
114
114
  HOST_SHAPED_SEGMENT = /\A(?:www\.)?(?:[\w-]+\.)+[a-z]{2,24}\z/i
115
115
  # Common file-extension suffixes excluded from host-shaped junk matching.
116
- FILE_EXTENSION_SEGMENT = /\.(?:pdf|jpe?g|png|gif|svg|webp|css|js|mjs|html?|xml|json|mp4|webm|zip|gz)\z/i
116
+ # Video files: a path ending in one is a media asset, never an article. Audio stays out (podcast items
117
+ # link their enclosure) and so do images (`/assets/image.jpg` is pinned as not junk).
118
+ MEDIA_FILE_EXTENSIONS = %w[mp4 webm flv mov m4v].freeze
119
+ # All file-extension suffixes, media included (single owner of the list).
120
+ FILE_EXTENSIONS = (%w[pdf jpe?g png gif svg webp css js mjs html? xml json zip gz] + MEDIA_FILE_EXTENSIONS).freeze
121
+ # Segment ending in a known file extension.
122
+ FILE_EXTENSION_SEGMENT = /\.(?:#{FILE_EXTENSIONS.join('|')})\z/i
123
+ # Segment ending in a video file extension.
124
+ MEDIA_FILE_SEGMENT = /\.(?:#{MEDIA_FILE_EXTENSIONS.join('|')})\z/i
125
+ # Document-style permalink: four or more words, then a numeric id of 5+ digits and a page extension
126
+ # (`/author/the-long-title_b_4421799.html`). A text-only slug is too common on product, forum and wiki pages.
127
+ DOCUMENT_SLUG_SEGMENT = /\A[\p{L}\d]+(?:[-_][\p{L}\d]+){3,}[-_]\d{5,}\.(?:html?|shtml|aspx?|php)\z/i
117
128
 
118
129
  # A single person-name slug: letters only (dated and id slugs carry digits), up to three words.
119
130
  NAME_SLUG = /\A\p{L}+(?:-\p{L}+){0,2}\z/
@@ -166,8 +177,8 @@ module Html2rss
166
177
  # @return [Boolean] true when the final path segment looks like a post slug
167
178
  def strong_post_suffix?
168
179
  @strong_post_suffix ||= segments.any? &&
169
- slug_last_segment? &&
170
- trusted_post_context?(segments.size - 1)
180
+ ((slug_last_segment? && trusted_post_context?(segments.size - 1)) ||
181
+ document_slug_route?)
171
182
  end
172
183
 
173
184
  # @return [Boolean] true when every path segment is utility chrome
@@ -190,7 +201,8 @@ module Html2rss
190
201
  def non_article_path?
191
202
  return @non_article_path if defined?(@non_article_path)
192
203
 
193
- @non_article_path = section_index? || profile_route? || (!strong_post_suffix? && non_article_word_present?)
204
+ @non_article_path = section_index? || profile_route? || media_file_route? ||
205
+ (!strong_post_suffix? && non_article_word_present?)
194
206
  end
195
207
 
196
208
  # @return [Boolean] true when the route is shallow and contains high-confidence noise
@@ -220,6 +232,10 @@ module Html2rss
220
232
  SEGMENT_SETS[:non_article].include?(joined) || joined.match?(PLAYER_SEGMENT)
221
233
  end
222
234
 
235
+ def media_file_route?
236
+ segments.any? && segments.last.match?(MEDIA_FILE_SEGMENT)
237
+ end
238
+
223
239
  # A people word that ends the path (an index) or is followed by one name slug (a profile page).
224
240
  def profile_route?
225
241
  return false if segments.empty?
@@ -290,6 +306,16 @@ module Html2rss
290
306
  end
291
307
  end
292
308
 
309
+ # A document slug needs no content/year context, unless a slug word is utility chrome or the path names a
310
+ # non-article destination (games, quiz, ...).
311
+ def document_slug_route?
312
+ return false if segments.size < 2 || leading_high_confidence_junk? || non_article_word_present?
313
+
314
+ last = segments.last
315
+ utility = SEGMENT_SETS.fetch(:utility) # vanity words are a subset of utility
316
+ last.match?(DOCUMENT_SLUG_SEGMENT) && last.split(/[-_.]/).none? { utility.include?(_1.downcase) }
317
+ end
318
+
293
319
  def slug_last_segment?
294
320
  last = segments.last
295
321
  last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)
@@ -43,7 +43,7 @@ module Html2rss
43
43
 
44
44
  # @return [HTTPX::Session]
45
45
  def base_ssrf_session
46
- @base_ssrf_session ||= base_session.plugin(:ssrf_filter)
46
+ @base_ssrf_session ||= base_session.plugin(NetworkGuardFilter)
47
47
  end
48
48
  # rubocop:enable ThreadSafety/ClassInstanceVariable
49
49
  end
@@ -27,8 +27,17 @@ module Html2rss
27
27
  IPAddr.new('::1/128'),
28
28
  IPAddr.new('fe80::/10'),
29
29
  IPAddr.new('fc00::/7'),
30
- IPAddr.new('ff00::/8')
30
+ IPAddr.new('ff00::/8'),
31
+ # RFC 8215 local-use NAT64 prefix: not globally reachable.
32
+ IPAddr.new('64:ff9b:1::/48'),
33
+ # Teredo (RFC 4380): client address is obfuscated, so the whole prefix is denied.
34
+ IPAddr.new('2001::/32')
31
35
  ].freeze
36
+ # RFC 6052 well-known NAT64 prefix; the last 32 bits embed an IPv4 address.
37
+ NAT64_PREFIX = IPAddr.new('64:ff9b::/96')
38
+ # 6to4 (RFC 3056); bits 16..47 embed an IPv4 address. Decoded only to deny private embeds;
39
+ # the HTTPX filter still denies all of 2002::/16 at connect.
40
+ SIX_TO_FOUR_PREFIX = IPAddr.new('2002::/16')
32
41
 
33
42
  ##
34
43
  # @param allow_private_networks [Boolean] whether private network targets are allowed
@@ -75,6 +84,31 @@ module Html2rss
75
84
  raise PrivateNetworkDenied, "Private network target denied for #{url}"
76
85
  end
77
86
 
87
+ ##
88
+ # Single owner of the IP allow/deny decision. IPv6 forms that embed an IPv4 address
89
+ # (IPv4-mapped, NAT64 64:ff9b::/96, 6to4) are allowed only if the embedded IPv4 is.
90
+ #
91
+ # @param address [IPAddr] resolved or remote address
92
+ # @return [Boolean] true when the address must be denied
93
+ def blocked_ip?(address)
94
+ native = address.native
95
+ return true if BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
96
+
97
+ embedded = embedded_ipv4(native)
98
+ !embedded.nil? && blocked_ip?(embedded)
99
+ end
100
+
101
+ ##
102
+ # @param address [IPAddr] native (non-mapped) address
103
+ # @return [IPAddr, nil] the IPv4 embedded in a NAT64 or 6to4 address, else nil
104
+ def embedded_ipv4(address)
105
+ if NAT64_PREFIX.include?(address)
106
+ IPAddr.new_ntoh(address.hton[12..])
107
+ elsif SIX_TO_FOUR_PREFIX.include?(address)
108
+ IPAddr.new_ntoh(address.hton[2, 4])
109
+ end
110
+ end
111
+
78
112
  private
79
113
 
80
114
  attr_reader :resolver
@@ -120,11 +154,6 @@ module Html2rss
120
154
  rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
121
155
  nil
122
156
  end
123
-
124
- def blocked_ip?(address)
125
- native = address.native
126
- BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
127
- end
128
157
  end
129
158
  end
130
159
  end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'httpx'
4
+ require 'httpx/plugins/ssrf_filter'
5
+
6
+ module Html2rss
7
+ class RequestService
8
+ ##
9
+ # HTTPX plugin layered on +:ssrf_filter+ (HTTPX 1.9.x, which offers no per-address predicate).
10
+ # +ssrf_filter+ denies the whole NAT64 prefix; here it is released via +safe_private_ranges+
11
+ # only together with a per-address check by {NetworkGuard} (embedded IPv4) and HTTPX's own
12
+ # IPv4 list on the embedded address. Either denying an address drops it (fail closed).
13
+ # Loading +:ssrf_filter+ first as a dependency keeps this module ahead of it in the
14
+ # ancestors; a spec pins that order.
15
+ #
16
+ # The release is gated at load: it is only applied on the tested HTTPX series and while the
17
+ # +ssrf_filter+ connection still routes resolution through +addresses=+. Otherwise HTTPX keeps
18
+ # denying all of 64:ff9b::/96 (fail closed) and a warning says NAT64 support is inactive.
19
+ #
20
+ # 6to4 is decoded by {NetworkGuard} only to deny private embedded IPv4; HTTPX itself still
21
+ # denies all of 2002::/16 at connect time.
22
+ module NetworkGuardFilter
23
+ # Strict guard instance: the plugin only exists on the SSRF-protected session.
24
+ GUARD = NetworkGuard.new(allow_private_networks: false)
25
+
26
+ # HTTPX series the release was verified against.
27
+ TESTED_HTTPX_SERIES = '1.9.'
28
+
29
+ class << self
30
+ # @param klass [Class] HTTPX session class being extended
31
+ # @return [void]
32
+ def load_dependencies(klass)
33
+ klass.plugin(:ssrf_filter)
34
+ end
35
+
36
+ # @param options [HTTPX::Options] session options
37
+ # @return [HTTPX::Options] options with the NAT64 prefix released to {NetworkGuard}
38
+ # when HTTPX is in the tested range; unchanged otherwise
39
+ def extra_options(options)
40
+ return options.merge(safe_private_ranges: [NetworkGuard::NAT64_PREFIX]) if release_supported?
41
+
42
+ warn_inactive
43
+ options
44
+ end
45
+
46
+ # @return [Boolean] true when the installed HTTPX is the tested series and still hooks +addresses=+
47
+ def release_supported?
48
+ HTTPX::VERSION.start_with?(TESTED_HTTPX_SERIES) &&
49
+ HTTPX::Plugins::SsrfFilter::ConnectionMethods.method_defined?(:addresses=)
50
+ end
51
+
52
+ # Logs once per process; carries no addresses.
53
+ # @return [void]
54
+ def warn_inactive
55
+ # rubocop:disable ThreadSafety/ClassInstanceVariable -- benign once-only log flag
56
+ return if @warned_inactive
57
+
58
+ @warned_inactive = true
59
+ # rubocop:enable ThreadSafety/ClassInstanceVariable
60
+ Log.warn({ event_name: 'network_guard.nat64_inactive', httpx_version: HTTPX::VERSION }.to_json)
61
+ end
62
+
63
+ # @param ipaddr [IPAddr]
64
+ # @return [Boolean] true when the guard or HTTPX's IPv4 list denies the address
65
+ def denied?(ipaddr)
66
+ return true if GUARD.blocked_ip?(ipaddr)
67
+
68
+ embedded = GUARD.embedded_ipv4(ipaddr)
69
+ !embedded.nil? && HTTPX::Plugins::SsrfFilter.unsafe_ip_address?(embedded)
70
+ end
71
+ end
72
+
73
+ # Connection hooks.
74
+ module ConnectionMethods
75
+ # @param addrs [Array<#address>] resolved entries
76
+ # @return [void]
77
+ # @raise [HTTPX::ServerSideRequestForgeryError] when no entry is allowed
78
+ def addresses=(addrs)
79
+ super(addrs.reject { |entry| NetworkGuardFilter.denied?(entry.address) })
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
@@ -19,9 +19,7 @@ module Html2rss
19
19
  return nil unless node
20
20
 
21
21
  href = node['href'] || node.text
22
- return nil if href.strip.empty?
23
-
24
- Html2rss::Url.from_relative(href, page_response.url)
22
+ Html2rss::Url.resolve_web(href, page_response.url)
25
23
  end
26
24
 
27
25
  # @param parsed_body [Nokogiri::HTML::Document]
@@ -32,7 +32,7 @@ module Html2rss
32
32
  return nil unless (next_url_path = config[:next_url_path])
33
33
 
34
34
  next_url = dig_path(json_data, next_url_path)
35
- Html2rss::Url.from_relative(next_url.to_s, origin_url) if next_url && !next_url.to_s.empty?
35
+ Html2rss::Url.resolve_web(next_url, origin_url)
36
36
  end
37
37
 
38
38
  # @param json_data [Hash]
@@ -13,9 +13,7 @@ module Html2rss
13
13
  # @return [String, nil]
14
14
  def next_page_url(page_response, page_number:) # rubocop:disable Lint/UnusedMethodArgument
15
15
  href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
16
- return nil if href.nil? || href.empty?
17
-
18
- Html2rss::Url.from_relative(href, page_response.url)
16
+ Html2rss::Url.resolve_web(href, page_response.url)
19
17
  end
20
18
  end
21
19
  end
@@ -59,6 +59,17 @@ module Html2rss
59
59
  Pager.for(pagination_config, session: self, initial_response:)
60
60
  end
61
61
 
62
+ ##
63
+ # Returns a session with the same strategy, policy, headers and origin but its own request budget, so a
64
+ # bounded optional step (article enrichment) cannot spend the listing's request slots. The wall clock is
65
+ # the remaining time of this session's budget (rounded up, at least one second).
66
+ #
67
+ # @param max_requests [Integer] request slots of the new budget
68
+ # @return [RequestSession]
69
+ def with_budget_allowance(max_requests)
70
+ self.class.new(context: allowance_context(max_requests), strategy:, logger:)
71
+ end
72
+
62
73
  ##
63
74
  # Executes a follow-up request sharing policy, headers, and budget.
64
75
  #
@@ -123,6 +134,15 @@ module Html2rss
123
134
 
124
135
  attr_reader :context, :strategy, :logger, :visited_urls
125
136
 
137
+ def allowance_context(max_requests)
138
+ remaining = context.budget.remaining_timeout_seconds
139
+ budget = RequestService::Budget.new(max_requests:, total_timeout_seconds: remaining && [remaining.ceil, 1].max)
140
+ RequestService::Context.new(
141
+ url:, headers: context.headers, request: context.request, policy: context.policy, budget:,
142
+ origin_url: context.origin_url
143
+ )
144
+ end
145
+
126
146
  def execute(request_context)
127
147
  RequestService.execute(request_context, strategy:).tap do |response|
128
148
  logger.debug(
@@ -54,9 +54,9 @@ module Html2rss
54
54
  ##
55
55
  # Retrieves and returns the normalized absolute URL.
56
56
  #
57
- # @return [Html2rss::Url, nil] The absolute URL, or nil when nothing matched or the +href+ is blank.
57
+ # @return [Html2rss::Url, nil] The absolute URL, or nil when nothing matched the +href+ is blank or unparseable.
58
58
  def call
59
- Url.from_relative(@href, @base_url) unless @href.strip.empty?
59
+ Url.resolve_web(@href, @base_url, web_only: false)
60
60
  end
61
61
  end
62
62
  end
@@ -33,7 +33,8 @@ module Html2rss
33
33
 
34
34
  url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[node_name]
35
35
  url = node[url_attribute]
36
- node[url_attribute] = Url.from_relative(url, @base_url).to_s
36
+ resolved = Url.resolve_web(url, @base_url, web_only: false)
37
+ node[url_attribute] = resolved.to_s if resolved
37
38
  end
38
39
  end
39
40
  end
@@ -48,9 +48,9 @@ module Html2rss
48
48
  end
49
49
 
50
50
  ##
51
- # @return [String]
51
+ # @return [String, nil] nil when +value+ is blank or unparseable
52
52
  def call
53
- Url.from_relative(value, context.base_url).to_s
53
+ Url.resolve_web(value, context.base_url, web_only: false)&.to_s
54
54
  end
55
55
  end
56
56
  end
@@ -452,10 +452,7 @@ module Html2rss
452
452
  end
453
453
 
454
454
  def default_item_url(item, base_url = @url)
455
- href = item['href'].to_s.strip
456
- return if href.empty?
457
-
458
- Url.from_relative(href, base_url)
455
+ Url.resolve_web(item['href'], base_url, web_only: false)
459
456
  end
460
457
 
461
458
  def item_env_for(item, base_url)
@@ -3,7 +3,8 @@
3
3
  module Html2rss
4
4
  Status = Data.define(
5
5
  :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
6
- :strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution
6
+ :strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution,
7
+ :enriched, :enrich_failed, :enrich_skipped_cross_origin
7
8
  )
8
9
 
9
10
  ##
@@ -12,7 +13,7 @@ module Html2rss
12
13
  # Exposed publicly via {FeedResult#status}. Safe to log without reading articles.
13
14
  # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
14
15
  # Tallies and counters are validated and frozen at construction (including Marshal load).
15
- class Status
16
+ class Status # rubocop:disable Metrics/ClassLength -- one reader and one validated counter per telemetry field
16
17
  # Raised when telemetry fields fail validation at construction.
17
18
  class InvalidError < Html2rss::ContractError; end
18
19
 
@@ -20,6 +21,10 @@ module Html2rss
20
21
  ALWAYS_REPORTED = %i[version dedup_dropped].freeze
21
22
  private_constant :ALWAYS_REPORTED
22
23
 
24
+ # Article-enrichment counters (see {Article::Enrichment::Result}).
25
+ ENRICHMENT_KEYS = %i[enriched enrich_failed enrich_skipped_cross_origin].freeze
26
+ private_constant :ENRICHMENT_KEYS
27
+
23
28
  class << self
24
29
  ##
25
30
  # Builds status from extracted articles and scrape telemetry.
@@ -34,11 +39,12 @@ module Html2rss
34
39
  # @param entry_url [String, nil] original channel URL when entry resolution ran
35
40
  # @param scrape_url [String, nil] effective scrape URL after resolution
36
41
  # @param entry_resolution [Html2rss::FeedResolution::Diag, Hash, nil] resolution diagnostics
42
+ # @param enrichment [Hash{Symbol => Integer}] {Article::Enrichment} tally (empty when off)
37
43
  # @return [Html2rss::Status]
38
44
  # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Status kwargs stay co-located
39
45
  def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0,
40
46
  strategy_attempts: [], admission_drops: {}, scrape_target: nil,
41
- entry_url: nil, scrape_url: nil, entry_resolution: nil)
47
+ entry_url: nil, scrape_url: nil, entry_resolution: nil, enrichment: {})
42
48
  tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
43
49
  new(
44
50
  version: Html2rss::VERSION,
@@ -50,7 +56,8 @@ module Html2rss
50
56
  admission_drops:,
51
57
  entry_url: entry_url || scrape_target&.entry_url,
52
58
  scrape_url: scrape_url || scrape_target&.effective_url,
53
- entry_resolution:
59
+ entry_resolution:,
60
+ **ENRICHMENT_KEYS.to_h { [_1, enrichment.fetch(_1, 0)] }
54
61
  )
55
62
  end
56
63
 
@@ -73,10 +80,14 @@ module Html2rss
73
80
  # @param entry_url [String, nil]
74
81
  # @param scrape_url [String, nil]
75
82
  # @param entry_resolution [Html2rss::FeedResolution::Diag, Hash, nil]
83
+ # @param enriched [Integer] items that gained a field from their article page
84
+ # @param enrich_failed [Integer] article fetches that failed or were blocked
85
+ # @param enrich_skipped_cross_origin [Integer] fetches skipped by the same-origin policy (subset of enrich_failed)
76
86
  # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
77
87
  def initialize(
78
88
  version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
79
- strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil
89
+ strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil,
90
+ enriched: 0, enrich_failed: 0, enrich_skipped_cross_origin: 0
80
91
  )
81
92
  dedup = Option.integer(:dedup_dropped, dedup_dropped, min: 0)
82
93
  attempts = Option.integer(:attempt_count, attempt_count, min: 0)
@@ -92,7 +103,10 @@ module Html2rss
92
103
  admission_drops: freeze_tallies(admission_drops),
93
104
  entry_url: freeze_optional_string(entry_url),
94
105
  scrape_url: freeze_optional_string(scrape_url),
95
- entry_resolution: freeze_entry_resolution(entry_resolution)
106
+ entry_resolution: freeze_entry_resolution(entry_resolution),
107
+ enriched: Option.integer(:enriched, enriched, min: 0),
108
+ enrich_failed: Option.integer(:enrich_failed, enrich_failed, min: 0),
109
+ enrich_skipped_cross_origin: Option.integer(:enrich_skipped_cross_origin, enrich_skipped_cross_origin, min: 0)
96
110
  )
97
111
  end
98
112
 
@@ -129,15 +143,18 @@ module Html2rss
129
143
 
130
144
  def marshal_dump
131
145
  [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
132
- admission_drops, entry_url, scrape_url, entry_resolution]
146
+ admission_drops, entry_url, scrape_url, entry_resolution, enriched, enrich_failed,
147
+ enrich_skipped_cross_origin]
133
148
  end
134
149
 
135
150
  def marshal_load(payload)
136
151
  version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
137
- strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution = payload
152
+ strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution,
153
+ enriched, enrich_failed, enrich_skipped_cross_origin = payload
138
154
  initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
139
155
  strategy_attempts:, admission_drops:, entry_url:, scrape_url:,
140
- entry_resolution:)
156
+ entry_resolution:, enriched: enriched || 0, enrich_failed: enrich_failed || 0,
157
+ enrich_skipped_cross_origin: enrich_skipped_cross_origin || 0)
141
158
  end
142
159
 
143
160
  def freeze_tallies(tallies)
data/lib/html2rss/url.rb CHANGED
@@ -45,6 +45,15 @@ module Html2rss
45
45
  SCHEME_PREFIX = /\A([a-z][a-z0-9+.-]*):/i
46
46
  private_constant :SCHEME_PREFIX
47
47
 
48
+ # One DNS label: 1-63 letters, digits, underscores and inner hyphens (IDN labels arrive punycoded).
49
+ DNS_LABEL = '[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?'
50
+ DNS_NAME = /\A(?:#{DNS_LABEL}\.)*#{DNS_LABEL}\z/i
51
+ OCTET = '(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
52
+ IPV4 = /\A(?:#{OCTET}\.){3}#{OCTET}\z/
53
+ NUMERIC_LAST_LABEL = /(?:\A|\.)\d+\z/
54
+ MAX_HOST_LENGTH = 253
55
+ private_constant :DNS_LABEL, :DNS_NAME, :OCTET, :IPV4, :NUMERIC_LAST_LABEL, :MAX_HOST_LENGTH
56
+
48
57
  ##
49
58
  # Creates a URL from a relative path and base URL.
50
59
  #
@@ -105,6 +114,8 @@ module Html2rss
105
114
  return unless match
106
115
 
107
116
  new(parse(match[0]).normalize)
117
+ rescue Addressable::URI::InvalidURIError, InvalidUrlError
118
+ nil
108
119
  end
109
120
 
110
121
  ##
@@ -160,12 +171,13 @@ module Html2rss
160
171
  #
161
172
  # @param ref [String, Html2rss::Url, nil] an absolute or relative URL, possibly padded with whitespace
162
173
  # @param base_url [String, Html2rss::Url, nil] the document the reference appears in
163
- # @return [Url, nil] the absolute http(s) URL, or nil when +ref+ is blank, unparseable or not http(s)
164
- def self.resolve_web(ref, base_url = nil)
174
+ # @param web_only [Boolean] keep only http(s); +false+ also keeps other schemes (+mailto:+ and the like)
175
+ # @return [Url, nil] the absolute URL, or nil when +ref+ is blank, unparseable or (with +web_only+) not http(s)
176
+ def self.resolve_web(ref, base_url = nil, web_only: true)
165
177
  return unless ref.to_s.match?(/\S/)
166
178
 
167
179
  url = from_relative(ref, base_url)
168
- url if url.web?
180
+ url if !web_only || url.web?
169
181
  rescue InvalidUrlError
170
182
  nil
171
183
  end
@@ -213,6 +225,8 @@ module Html2rss
213
225
  #
214
226
  # @param uri [Addressable::URI] the normalized Addressable::URI (internal use only)
215
227
  def initialize(uri)
228
+ raise InvalidUrlError, 'URL host is invalid' unless valid_host?(uri.host)
229
+
216
230
  @uri = uri
217
231
  @path_segments = path.split('/').reject(&:empty?).freeze
218
232
  freeze
@@ -271,6 +285,14 @@ module Html2rss
271
285
  # @return [Array<String>] normalized path segments
272
286
  attr_reader :path_segments
273
287
 
288
+ ##
289
+ # Path segments as plain text: percent-decoded, scrubbed and NFC-normalized (for matching words and digits).
290
+ #
291
+ # @return [Array<String>] decoded path segments
292
+ def decoded_path_segments
293
+ path_segments.map { Addressable::URI.unencode_component(_1).force_encoding(Encoding::UTF_8).scrub.unicode_normalize(:nfc) }
294
+ end
295
+
274
296
  ##
275
297
  # Returns a copy of the URL with the provided path.
276
298
  #
@@ -440,6 +462,35 @@ module Html2rss
440
462
 
441
463
  def tracking_param?(pair) = TRACKING_PARAM.match?(pair.partition('=').first)
442
464
 
465
+ ##
466
+ # The one host rule: no host (relative or opaque URL), a DNS name, an IPv4 literal, or a bracketed IPv6
467
+ # literal that parses. Addressable's +normalize+ already strips a trailing dot; a Unicode host is checked in
468
+ # its punycode form. An IPv4 octet has no leading zero (+01.2.3.4+, +0177.0.0.1+): readers disagree on
469
+ # whether that is decimal or octal, the classic shape of an address-guard bypass.
470
+ #
471
+ # @param host [String, nil]
472
+ # @return [Boolean] truthy when valid
473
+ def valid_host?(host)
474
+ return true if host.nil? || host.empty?
475
+
476
+ host = Addressable::IDNA.to_ascii(host) unless host.ascii_only?
477
+ return ipv6_literal?(host) if host.start_with?('[')
478
+
479
+ dns_name_or_ipv4?(host)
480
+ end
481
+
482
+ def ipv6_literal?(host)
483
+ IPAddr.new(host, Socket::AF_INET6)
484
+ rescue IPAddr::Error
485
+ false
486
+ end
487
+
488
+ def dns_name_or_ipv4?(name)
489
+ return IPV4.match?(name) if NUMERIC_LAST_LABEL.match?(name)
490
+
491
+ name.length <= MAX_HOST_LENGTH && DNS_NAME.match?(name)
492
+ end
493
+
443
494
  def ip_address_host?
444
495
  !IPAddr.new(host).nil?
445
496
  rescue IPAddr::Error
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Html2rss
4
4
  # Current application version.
5
- VERSION = '0.34.1'
5
+ VERSION = '0.35.0'
6
6
  public_constant :VERSION
7
7
  end
@@ -855,6 +855,26 @@
855
855
  }
856
856
  },
857
857
  "required": []
858
+ },
859
+ "enrich": {
860
+ "type": "object",
861
+ "properties": {
862
+ "article_metadata": {
863
+ "type": "boolean",
864
+ "not": {
865
+ "type": "null"
866
+ }
867
+ },
868
+ "max_fetches": {
869
+ "type": "integer",
870
+ "not": {
871
+ "type": "null"
872
+ },
873
+ "exclusiveMinimum": 0,
874
+ "maximum": 10
875
+ }
876
+ },
877
+ "required": []
858
878
  }
859
879
  },
860
880
  "required": [
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: html2rss
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.34.1
4
+ version: 0.35.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Gil Desmarais
@@ -270,6 +270,7 @@ files:
270
270
  - lib/html2rss/article.rb
271
271
  - lib/html2rss/article/deduplicator.rb
272
272
  - lib/html2rss/article/enclosure.rb
273
+ - lib/html2rss/article/enrichment.rb
273
274
  - lib/html2rss/article/title.rb
274
275
  - lib/html2rss/auto_source.rb
275
276
  - lib/html2rss/auto_source/README.md
@@ -372,6 +373,7 @@ files:
372
373
  - lib/html2rss/html/article_rules/description.rb
373
374
  - lib/html2rss/html/article_rules/enclosure.rb
374
375
  - lib/html2rss/html/article_rules/image.rb
376
+ - lib/html2rss/html/article_rules/url_date.rb
375
377
  - lib/html2rss/html/card_walk.rb
376
378
  - lib/html2rss/html/feed_link.rb
377
379
  - lib/html2rss/html/navigator.rb
@@ -424,6 +426,7 @@ files:
424
426
  - lib/html2rss/request_service/httpx_strategy.rb
425
427
  - lib/html2rss/request_service/local_file_strategy.rb
426
428
  - lib/html2rss/request_service/network_guard.rb
429
+ - lib/html2rss/request_service/network_guard_filter.rb
427
430
  - lib/html2rss/request_service/policy.rb
428
431
  - lib/html2rss/request_service/replay_breaker.rb
429
432
  - lib/html2rss/request_service/response.rb
@@ -507,7 +510,7 @@ licenses:
507
510
  - MIT
508
511
  metadata:
509
512
  allowed_push_host: https://rubygems.org
510
- changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.34.1
513
+ changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.35.0
511
514
  rubygems_mfa_required: 'true'
512
515
  rdoc_options: []
513
516
  require_paths: