html2rss 0.34.1 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -0
- data/lib/html2rss/article/enrichment.rb +179 -0
- data/lib/html2rss/article.rb +12 -0
- data/lib/html2rss/auto_source/scraper/sitemap.rb +1 -1
- data/lib/html2rss/capture.rb +2 -1
- data/lib/html2rss/config/validator.rb +7 -0
- data/lib/html2rss/config.rb +2 -0
- data/lib/html2rss/feed_pipeline.rb +49 -10
- data/lib/html2rss/html/article_extractor/image_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor.rb +1 -1
- data/lib/html2rss/html/article_rules/date.rb +11 -0
- data/lib/html2rss/html/article_rules/url_date.rb +78 -0
- data/lib/html2rss/html/sst_article_extractor.rb +2 -2
- data/lib/html2rss/link_destination/destination_facts.rb +1 -9
- data/lib/html2rss/link_destination/path_classifier.rb +31 -5
- data/lib/html2rss/request_service/httpx_strategy.rb +1 -1
- data/lib/html2rss/request_service/network_guard.rb +35 -6
- data/lib/html2rss/request_service/network_guard_filter.rb +84 -0
- data/lib/html2rss/request_session/pager/custom_selector.rb +1 -3
- data/lib/html2rss/request_session/pager/json_cursor.rb +1 -1
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -3
- data/lib/html2rss/request_session.rb +20 -0
- data/lib/html2rss/selectors/extractors/href.rb +2 -2
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +2 -1
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +2 -2
- data/lib/html2rss/selectors.rb +1 -4
- data/lib/html2rss/status.rb +26 -9
- data/lib/html2rss/url.rb +54 -3
- data/lib/html2rss/version.rb +1 -1
- data/schema/html2rss-config.schema.json +20 -0
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: db36711f78369ba0783ffe4203df047b431224926ce3ea8528985f0498237653
|
|
4
|
+
data.tar.gz: c819bec563f9ec056d909861b6c62ef85afcc7411906404c88e731248c3b17a5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 54dedcfaea4d2d521d6dcc9a8850513f9e946c10c3f951c569ba1bc21662791be5d727bac5d27d654bb7c3941378e51887bdc05f3aa301b738dd12bc17b946ae
|
|
7
|
+
data.tar.gz: a9c66a90a7ce89be190ed0179c72833bdbbf0c9ef014e27dfb639cef76de35d47a95d29713ce59ef79f3370dabad7b1a5a4585f8015138a319d476a28fc63b06
|
data/README.md
CHANGED
|
@@ -184,6 +184,20 @@ docker compose -f docker-compose.botasaurus.yml up -d
|
|
|
184
184
|
|
|
185
185
|
Set `BOTASAURUS_SCRAPER_URL` to `http://127.0.0.1:4010` and use strategy `botasaurus` in MCP tools, Capture, or the CLI.
|
|
186
186
|
|
|
187
|
+
## Article-page enrichment
|
|
188
|
+
|
|
189
|
+
Listing pages often lack a date, author or image. With `enrich` set, the feed build fetches the item's own page and fills only those missing fields from its JSON-LD (the page's own `author` and `datePublished`) and meta tags (`og:image`, `article:published_time`, `article:author`). Values already found on the listing are never overwritten, and an article-page date wins over a date guessed from the URL. Titles, descriptions and body text are never touched.
|
|
190
|
+
|
|
191
|
+
```yaml
|
|
192
|
+
enrich:
|
|
193
|
+
article_metadata: true # default: off
|
|
194
|
+
max_fetches: 10 # default and hard maximum: 10
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Cost: up to `max_fetches` extra requests per build, on top of the normal request budget, using the same request strategy. Links must stay on the feed's origin; others are skipped. A failed or blocked fetch leaves the item unchanged and never fails the build.
|
|
198
|
+
|
|
199
|
+
`--explain` prints three counts when enrichment ran: `enriched` (items that gained a field), `enrich_failed` (failed or blocked fetches, including cross-origin skips) and `enrich_skipped_cross_origin` (the cross-origin subset).
|
|
200
|
+
|
|
187
201
|
## Request Strategies
|
|
188
202
|
|
|
189
203
|
| Strategy | Description |
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'cgi'
|
|
4
|
+
|
|
5
|
+
module Html2rss
|
|
6
|
+
class Article
|
|
7
|
+
##
|
|
8
|
+
# Opt-in feed-build step and the single owner of "metadata from an article page": for items missing
|
|
9
|
+
# +published_at+, +author+ or +image+ it fetches the item's own page and fills only those fields from the
|
|
10
|
+
# page's JSON-LD and meta tags. Listing values are never overwritten; body text and description are never read.
|
|
11
|
+
#
|
|
12
|
+
# Fetches go through a {RequestSession} sub-allowance, so strategy, policy (same-origin rule, network guard,
|
|
13
|
+
# response size cap) and headers match the listing fetch. The allowance is separate from the listing
|
|
14
|
+
# budget, so one build makes at most {MAX_TOTAL_REQUESTS} requests: the listing budget
|
|
15
|
+
# ({RequestService::Policy::MAX_REQUESTS_CEILING}) plus {MAX_FETCHES}. A blocked or failed fetch leaves the
|
|
16
|
+
# item unchanged and is counted; enrichment never fails the build.
|
|
17
|
+
class Enrichment # rubocop:disable Metrics/ClassLength -- fetch, parse and fill share one owner
|
|
18
|
+
# Hard maximum and default number of article fetches per feed build.
|
|
19
|
+
MAX_FETCHES = 10
|
|
20
|
+
# Upper bound on requests one build can make: listing budget plus the enrichment allowance.
|
|
21
|
+
MAX_TOTAL_REQUESTS = RequestService::Policy::MAX_REQUESTS_CEILING + MAX_FETCHES
|
|
22
|
+
# Per field an article page can supply: how to read it from an article and how to set it.
|
|
23
|
+
FIELDS = {
|
|
24
|
+
published_at: [lambda(&:published_at), ->(article, value) { article.with_published_at(value) }],
|
|
25
|
+
author: [lambda(&:author), ->(article, value) { article.with_author(value) }],
|
|
26
|
+
image: [lambda(&:image), ->(article, value) { article.with_image(value) }]
|
|
27
|
+
}.freeze
|
|
28
|
+
# Schema.org keys that are weaker sources than a Thing's own author and datePublished.
|
|
29
|
+
WEAK_KEYS = %i[publisher hiringOrganization organizer datePosted dateCreated dateModified uploadDate].freeze
|
|
30
|
+
# Schema.org types that are the page's article rather than a related item.
|
|
31
|
+
ARTICLE_TYPES = Set['Article', 'NewsArticle', 'BlogPosting'].freeze
|
|
32
|
+
# Meta tag keys per field, in precedence order.
|
|
33
|
+
META_KEYS = {
|
|
34
|
+
published_at: %w[article:published_time],
|
|
35
|
+
author: %w[article:author author],
|
|
36
|
+
image: %w[og:image]
|
|
37
|
+
}.freeze
|
|
38
|
+
# Result of one run. +tally+ keys: enriched (items that gained a field), enrich_failed (fetches that
|
|
39
|
+
# failed or were blocked, including cross-origin skips), enrich_skipped_cross_origin (the policy-denied subset).
|
|
40
|
+
Result = Data.define(:articles, :tally)
|
|
41
|
+
|
|
42
|
+
class << self
|
|
43
|
+
##
|
|
44
|
+
# @param articles [Array<Html2rss::Article>] deduplicated articles
|
|
45
|
+
# @param request_session [Html2rss::RequestSession] the listing session (strategy and policy are reused)
|
|
46
|
+
# @param max_fetches [Integer] article fetch cap, capped at {MAX_FETCHES}
|
|
47
|
+
# @return [Result] the articles (unchanged ones kept in place) and the enrichment tally
|
|
48
|
+
def call(articles, request_session:, max_fetches: MAX_FETCHES)
|
|
49
|
+
cap = [max_fetches, MAX_FETCHES].min
|
|
50
|
+
new(request_session.with_budget_allowance(cap), cap).call(articles)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# @param session [Html2rss::RequestSession] session with the enrichment request allowance
|
|
55
|
+
# @param cap [Integer] maximum article fetches
|
|
56
|
+
def initialize(session, cap)
|
|
57
|
+
@session = session
|
|
58
|
+
@cap = cap
|
|
59
|
+
@tally = {}
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# @param articles [Array<Html2rss::Article>]
|
|
63
|
+
# @return [Result]
|
|
64
|
+
def call(articles)
|
|
65
|
+
budget = @cap
|
|
66
|
+
enriched = articles.map do |article|
|
|
67
|
+
next article unless budget.positive? && candidate?(article)
|
|
68
|
+
|
|
69
|
+
budget -= 1
|
|
70
|
+
enrich(article)
|
|
71
|
+
end
|
|
72
|
+
Result.new(articles: enriched, tally: @tally)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def candidate?(article)
|
|
78
|
+
article.url && FIELDS.each_value.any? { |read, _| read.call(article).nil? }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def enrich(article)
|
|
82
|
+
response = fetch(article.url)
|
|
83
|
+
return failed(article) unless page?(response)
|
|
84
|
+
|
|
85
|
+
fill(article, page_metadata(response))
|
|
86
|
+
rescue RequestService::CrossOriginFollowUpDenied
|
|
87
|
+
count(:enrich_skipped_cross_origin)
|
|
88
|
+
failed(article)
|
|
89
|
+
rescue Html2rss::ContractError
|
|
90
|
+
raise
|
|
91
|
+
rescue Html2rss::Error, JSON::ParserError
|
|
92
|
+
failed(article)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Sets only the fields the article lacks; counts the article when any was set.
|
|
96
|
+
def fill(article, found)
|
|
97
|
+
filled = FIELDS.reduce(article) do |current, (field, (read, write))|
|
|
98
|
+
read.call(current).nil? && found[field] ? write.call(current, found.fetch(field)) : current
|
|
99
|
+
end
|
|
100
|
+
count(:enriched) unless filled.equal?(article)
|
|
101
|
+
filled
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def count(key)
|
|
105
|
+
@tally[key] = @tally.fetch(key, 0) + 1
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def failed(article)
|
|
109
|
+
count(:enrich_failed)
|
|
110
|
+
article
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def fetch(url)
|
|
114
|
+
@session.follow_up(url:, relation: :auto_source, origin_url: @session.url)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def page?(response)
|
|
118
|
+
(response.status.nil? || response.status.between?(200, 299)) && response.html_response?
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def page_metadata(response)
|
|
122
|
+
document = response.parsed_body
|
|
123
|
+
things = jsonld_things(document, response.url)
|
|
124
|
+
FIELDS.keys.to_h { |field| [field, jsonld_value(things, field) || meta_value(document, field, response.url)] }
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# The page's own article Things, best first: the Thing whose url matches the page, then article-typed
|
|
128
|
+
# ones, then document order. Each Thing is built without the weaker sources {WEAK_KEYS} (Thing#author
|
|
129
|
+
# would fall back to publisher; Thing#published_at to modified/upload dates).
|
|
130
|
+
def jsonld_things(document, url)
|
|
131
|
+
things = document.css('script[type="application/ld+json"]').flat_map { thing_objects(_1) }
|
|
132
|
+
.map { AutoSource::Scraper::Schema::Thing.new(_1.except(*WEAK_KEYS), url:) }
|
|
133
|
+
things.group_by { thing_rank(_1, url) }.sort.flat_map(&:last)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def thing_objects(script)
|
|
137
|
+
schema = AutoSource::Scraper::Schema
|
|
138
|
+
schema.from_script(script).select { schema.scraper_for_schema_object(_1) == schema::Thing }
|
|
139
|
+
rescue JSON::ParserError
|
|
140
|
+
[]
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def thing_rank(thing, page_url)
|
|
144
|
+
return 0 if comparable(thing.url) == comparable(page_url)
|
|
145
|
+
|
|
146
|
+
types = AutoSource::Scraper::Schema.normalize_types(thing.schema_object[:@type])
|
|
147
|
+
types.intersect?(ARTICLE_TYPES) ? 1 : 2
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def comparable(url) = url.to_s.chomp('/')
|
|
151
|
+
|
|
152
|
+
def jsonld_value(things, field)
|
|
153
|
+
things.each do |thing|
|
|
154
|
+
value = thing_value(thing, field).to_s.strip
|
|
155
|
+
return value unless value.empty?
|
|
156
|
+
end
|
|
157
|
+
nil
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def thing_value(thing, field)
|
|
161
|
+
case field
|
|
162
|
+
when :published_at then thing.published_at
|
|
163
|
+
when :author then thing.author
|
|
164
|
+
else thing.image
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def meta_value(document, field, base)
|
|
169
|
+
META_KEYS.fetch(field).each do |key|
|
|
170
|
+
value = document.at_css("meta[property='#{key}'], meta[name='#{key}']")&.[]('content').to_s.strip
|
|
171
|
+
next if value.empty? || (field == :author && value.include?('://'))
|
|
172
|
+
|
|
173
|
+
return field == :image ? Url.from_relative(value, base).to_s : value
|
|
174
|
+
end
|
|
175
|
+
nil
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
data/lib/html2rss/article.rb
CHANGED
|
@@ -60,6 +60,18 @@ module Html2rss
|
|
|
60
60
|
linked? && (!title.to_s.empty? || !description.to_s.empty?) && !id.to_s.empty?
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
+
# @param value [DateTime, String, nil] replacement publication timestamp
|
|
64
|
+
# @return [Article] a copy of this article carrying +value+ as published_at
|
|
65
|
+
def with_published_at(value) = self.class.new(**@to_h, published_at: value)
|
|
66
|
+
|
|
67
|
+
# @param value [String, nil] replacement author name
|
|
68
|
+
# @return [Article] a copy of this article carrying +value+ as author
|
|
69
|
+
def with_author(value) = self.class.new(**@to_h, author: value)
|
|
70
|
+
|
|
71
|
+
# @param value [String, Html2rss::Url, nil] replacement image URL
|
|
72
|
+
# @return [Article] a copy of this article carrying +value+ as image
|
|
73
|
+
def with_image(value) = self.class.new(**@to_h, image: value)
|
|
74
|
+
|
|
63
75
|
# @param new_url [String, Html2rss::Url] replacement article URL
|
|
64
76
|
# @return [Article] a copy of this article carrying +new_url+
|
|
65
77
|
def with_url(new_url) = self.class.new(**@to_h, url: new_url)
|
|
@@ -151,7 +151,7 @@ module Html2rss
|
|
|
151
151
|
def target_sitemap_url
|
|
152
152
|
link_tag = parsed_body.at_css(SITEMAP_LINK_SELECTOR)
|
|
153
153
|
href = link_tag ? link_tag['href'] : '/sitemap.xml'
|
|
154
|
-
Html2rss::Url.
|
|
154
|
+
Html2rss::Url.resolve_web(href, url) || Html2rss::Url.resolve_web('/sitemap.xml', url)
|
|
155
155
|
end
|
|
156
156
|
|
|
157
157
|
def article_from_entry(entry)
|
data/lib/html2rss/capture.rb
CHANGED
|
@@ -329,7 +329,8 @@ module Html2rss
|
|
|
329
329
|
end
|
|
330
330
|
|
|
331
331
|
def find_matching_article(segment, articles_by_url)
|
|
332
|
-
resolved = Url.
|
|
332
|
+
return unless (resolved = Url.resolve_web(segment.primary_link.attrs.href, @url, web_only: false))
|
|
333
|
+
|
|
333
334
|
articles_by_url[resolved.to_s] || articles_by_url.values.find { |a| fuzzy_url_match?(a.url, resolved) }
|
|
334
335
|
end
|
|
335
336
|
|
|
@@ -93,6 +93,12 @@ module Html2rss
|
|
|
93
93
|
optional(:local_file_path).filled(:string)
|
|
94
94
|
end
|
|
95
95
|
|
|
96
|
+
# Contract for the opt-in `enrich` section (article-page metadata for items missing date, author or image).
|
|
97
|
+
EnrichConfig = Dry::Schema.Params do
|
|
98
|
+
optional(:article_metadata).filled(:bool)
|
|
99
|
+
optional(:max_fetches).filled(:integer, gt?: 0, lteq?: Html2rss::Article::Enrichment::MAX_FETCHES)
|
|
100
|
+
end
|
|
101
|
+
|
|
96
102
|
params do
|
|
97
103
|
optional(:strategy).filled(:symbol)
|
|
98
104
|
required(:channel).hash(ChannelConfig)
|
|
@@ -103,6 +109,7 @@ module Html2rss
|
|
|
103
109
|
optional(:selectors).hash
|
|
104
110
|
optional(:dynamic_params_error).maybe(:string)
|
|
105
111
|
optional(:request).hash(RequestConfig)
|
|
112
|
+
optional(:enrich).hash(EnrichConfig)
|
|
106
113
|
end
|
|
107
114
|
|
|
108
115
|
rule(:headers) do
|
data/lib/html2rss/config.rb
CHANGED
|
@@ -316,6 +316,8 @@ module Html2rss
|
|
|
316
316
|
|
|
317
317
|
# @return [Hash{Symbol => Object, nil}] selectors configuration
|
|
318
318
|
def selectors = config[:selectors]
|
|
319
|
+
# @return [Hash{Symbol => Object}, nil] article-page enrichment options (+article_metadata+, +max_fetches+)
|
|
320
|
+
def enrich = config[:enrich]
|
|
319
321
|
# @return [Hash{Symbol => Object, nil}] auto-source configuration
|
|
320
322
|
def auto_source = config[:auto_source]
|
|
321
323
|
|
|
@@ -6,22 +6,37 @@ module Html2rss
|
|
|
6
6
|
#
|
|
7
7
|
# {include:file:lib/html2rss/feed_pipeline/README.md}
|
|
8
8
|
class FeedPipeline # rubocop:disable Metrics/ClassLength -- outcome + collect seams stay co-located
|
|
9
|
+
PipelineOutcome = Data.define(
|
|
10
|
+
:response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
|
|
11
|
+
:admission_drops, :scrape_target, :entry_resolution, :feed_channel, :enrichment
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
Extraction = Data.define(:articles, :dedup_dropped, :admission_drops, :feed_channel, :enrichment)
|
|
15
|
+
|
|
9
16
|
# Scrape-finished facts after request + extraction + dedup (before Channel/Status materialize).
|
|
10
17
|
# selected_strategy: set on :auto success; nil otherwise.
|
|
11
18
|
# attempt_count: auto attempts attempted; 0 outside :auto.
|
|
12
19
|
# strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
|
|
13
20
|
# feed_channel: channel facts of the native feed followed ({Syndication::ChannelFacts}); nil without one.
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
21
|
+
# enrichment: {Article::Enrichment} tally (empty when enrichment is off).
|
|
22
|
+
class PipelineOutcome
|
|
23
|
+
# @param enrichment [Hash{Symbol => Integer}] defaults to an empty tally (enrichment off)
|
|
24
|
+
# @param fields [Hash{Symbol => Object}] the remaining members
|
|
25
|
+
# @option fields [Object] :articles the other members, as named above
|
|
26
|
+
def initialize(enrichment: {}, **fields) = super
|
|
27
|
+
end
|
|
18
28
|
|
|
19
29
|
# What one response yields after extraction and dedup ({#extract}); its fields are the
|
|
20
30
|
# {PipelineOutcome} fields of the same names.
|
|
21
31
|
# articles: unique articles. dedup_dropped: articles removed by deduplication.
|
|
22
32
|
# admission_drops: Cleanup drop tallies. feed_channel: the followed native feed's channel facts
|
|
23
|
-
# ({Syndication::ChannelFacts}); nil without one.
|
|
24
|
-
Extraction
|
|
33
|
+
# ({Syndication::ChannelFacts}); nil without one. enrichment: {Article::Enrichment} tally (empty when off).
|
|
34
|
+
class Extraction
|
|
35
|
+
# @param enrichment [Hash{Symbol => Integer}] defaults to an empty tally (enrichment off)
|
|
36
|
+
# @param fields [Hash{Symbol => Object}] the remaining members
|
|
37
|
+
# @option fields [Object] :articles the other members, as named above
|
|
38
|
+
def initialize(enrichment: {}, **fields) = super
|
|
39
|
+
end
|
|
25
40
|
|
|
26
41
|
##
|
|
27
42
|
# @param raw_config [Hash{Symbol => Object}] user-provided feed config
|
|
@@ -82,14 +97,38 @@ module Html2rss
|
|
|
82
97
|
# @return [Extraction] unique articles with their dedup and admission drops and feed channel facts
|
|
83
98
|
def extract(config:, response:, request_session:)
|
|
84
99
|
collected, admission_drops, feed_channel = collect_articles(config:, response:, request_session:)
|
|
85
|
-
|
|
86
|
-
|
|
100
|
+
deduped = Article::Deduplicator.new(collected).call
|
|
101
|
+
enriched, enrichment = enrich(deduped, config:, request_session:)
|
|
102
|
+
unique = with_url_dates(enriched, config.time_zone)
|
|
103
|
+
Extraction.new(articles: unique, dedup_dropped: collected.size - deduped.size, admission_drops:, feed_channel:,
|
|
104
|
+
enrichment:)
|
|
87
105
|
end
|
|
88
106
|
|
|
89
107
|
private
|
|
90
108
|
|
|
91
109
|
attr_reader :raw_config
|
|
92
110
|
|
|
111
|
+
# Opt-in article-page metadata. Runs before {#with_url_dates} so a real article-page date wins over a URL date.
|
|
112
|
+
def enrich(articles, config:, request_session:)
|
|
113
|
+
return [articles, {}] unless (options = config.enrich)&.fetch(:article_metadata, false)
|
|
114
|
+
|
|
115
|
+
result = Article::Enrichment.call(
|
|
116
|
+
articles, request_session:, max_fetches: options.fetch(:max_fetches, Article::Enrichment::MAX_FETCHES)
|
|
117
|
+
)
|
|
118
|
+
[result.articles, result.tally]
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# The single owner of the URL-path date fallback: it runs on the final articles, after cross-scraper
|
|
122
|
+
# merge and dedup, so no earlier stage (card walk, merge, scoring) ever sees a URL date.
|
|
123
|
+
def with_url_dates(articles, time_zone)
|
|
124
|
+
articles.map do |article|
|
|
125
|
+
next article if article.published_at
|
|
126
|
+
|
|
127
|
+
date = Html::ArticleRules::UrlDate.call(article.url, time_zone:)
|
|
128
|
+
date ? article.with_published_at(date) : article
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
93
132
|
def pipeline_outcome_for(config)
|
|
94
133
|
plan = StrategyPlan.resolve(config.strategy)
|
|
95
134
|
resources = RuntimePolicy.resources_for(config)
|
|
@@ -181,7 +220,7 @@ module Html2rss
|
|
|
181
220
|
[articles, {}, feed.channel]
|
|
182
221
|
end
|
|
183
222
|
|
|
184
|
-
def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
|
|
223
|
+
def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
185
224
|
channel = Channel.from_response(outcome.response, overrides: config.channel, feed: outcome.feed_channel)
|
|
186
225
|
status = Status.build(
|
|
187
226
|
articles: outcome.articles,
|
|
@@ -191,7 +230,7 @@ module Html2rss
|
|
|
191
230
|
strategy_attempts: outcome.strategy_attempts,
|
|
192
231
|
admission_drops: outcome.admission_drops,
|
|
193
232
|
scrape_target: outcome.scrape_target,
|
|
194
|
-
entry_resolution: outcome.entry_resolution
|
|
233
|
+
entry_resolution: outcome.entry_resolution, enrichment: outcome.enrichment
|
|
195
234
|
)
|
|
196
235
|
FeedResult.new(channel:, articles: outcome.articles, status:, stylesheets: config.stylesheets)
|
|
197
236
|
end
|
|
@@ -62,7 +62,7 @@ module Html2rss
|
|
|
62
62
|
def extract_url
|
|
63
63
|
@extract_url ||= begin
|
|
64
64
|
href = selected_anchor&.[]('href').to_s
|
|
65
|
-
Url.
|
|
65
|
+
Url.resolve_web(href.split('#').first, base_url, web_only: false)
|
|
66
66
|
end
|
|
67
67
|
end
|
|
68
68
|
|
|
@@ -31,6 +31,17 @@ module Html2rss
|
|
|
31
31
|
values.min
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
+
##
|
|
35
|
+
# @param time_zone [String] channel time zone
|
|
36
|
+
# @return [::Date] today's calendar day in +time_zone+
|
|
37
|
+
def local_today(time_zone:) = timezone_for(time_zone).now.to_date
|
|
38
|
+
|
|
39
|
+
##
|
|
40
|
+
# @param date [::Date] calendar day
|
|
41
|
+
# @param time_zone [String] channel time zone
|
|
42
|
+
# @return [DateTime] midnight of +date+ in +time_zone+
|
|
43
|
+
def at_midnight(date, time_zone:) = localize(DateTime.new(date.year, date.month, date.day), time_zone)
|
|
44
|
+
|
|
34
45
|
private
|
|
35
46
|
|
|
36
47
|
def parse_attr(value, time_zone:)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'date'
|
|
4
|
+
|
|
5
|
+
module Html2rss
|
|
6
|
+
module Html
|
|
7
|
+
module ArticleRules
|
|
8
|
+
##
|
|
9
|
+
# Sole owner of "publication date from an article URL path". It is applied once, on the final
|
|
10
|
+
# article, only when no other date source filled +published_at+. Precision over recall.
|
|
11
|
+
#
|
|
12
|
+
# Algorithm adapted from newspaper4k's DATE_REGEX (MIT), tightened to two shapes, each year-first,
|
|
13
|
+
# starting a path segment and followed by a non-empty slug: +/YYYY/M/D/slug+ (day is its own segment)
|
|
14
|
+
# and +YYYY-MM-DD-slug+ or +YYYY_MM_DD/slug+ (one in-segment separator, two-digit month and day).
|
|
15
|
+
# Numeric ids (+58201224+), day-less paths (+/2020/06/88083+), bare +/2024/01/01/+,
|
|
16
|
+
# +sku-2023-10-15-blue+, WordPress +/2024/05/10-best-laptops+ and CalVer +/docs/2024.1.2-notes+
|
|
17
|
+
# therefore yield nothing. The result is day precision: midnight in the channel time zone.
|
|
18
|
+
#
|
|
19
|
+
# Accepted residual: evergreen or past-event pages whose path carries a date get that date, since
|
|
20
|
+
# a path cannot tell publication from event date.
|
|
21
|
+
module UrlDate
|
|
22
|
+
# Calendar-year rule (bounded, 1995..today), unlike PathClassifier::YEARISH_SEGMENT's routing shape.
|
|
23
|
+
# Earliest year accepted as a publication year.
|
|
24
|
+
MIN_YEAR = 1995
|
|
25
|
+
# A whole segment that is a plausible-looking four-digit year.
|
|
26
|
+
YEAR = /\A(?:19|20)\d{2}\z/
|
|
27
|
+
# A whole segment of one or two digits (month or day).
|
|
28
|
+
DAY = /\A\d{1,2}\z/
|
|
29
|
+
# One segment +YYYY-MM-DD+ or +YYYY_MM_DD+ (same separator), then +-+/+_+ and a slug, or its end
|
|
30
|
+
# (a later segment must then be the slug).
|
|
31
|
+
JOINED = /\A((?:19|20)\d{2})([-_])(\d{2})\2(\d{2})(?:[-_].|(\z))/m
|
|
32
|
+
|
|
33
|
+
class << self
|
|
34
|
+
##
|
|
35
|
+
# @param url [Html2rss::Url, String, nil] article URL (only its path is read)
|
|
36
|
+
# @param time_zone [String] channel time zone
|
|
37
|
+
# @return [DateTime, nil] local midnight of the one unambiguous path date
|
|
38
|
+
def call(url, time_zone:)
|
|
39
|
+
dates = path_dates(Url.from_absolute(url).decoded_path_segments, time_zone:)
|
|
40
|
+
return unless dates.one?
|
|
41
|
+
|
|
42
|
+
Date.at_midnight(dates.first, time_zone:)
|
|
43
|
+
rescue Url::InvalidUrlError
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def path_dates(segments, time_zone:)
|
|
50
|
+
latest = Date.local_today(time_zone:) + 1
|
|
51
|
+
candidates = segments.each_index.filter_map { candidate(segments.drop(_1)) }
|
|
52
|
+
candidates.map { plausible(_1.map(&:to_i), latest) }.uniq
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# +/YYYY/M/D/slug+ is four consecutive segments; the joined shape is one segment, so a decoded
|
|
56
|
+
# +%2F+ inside a segment never makes a boundary.
|
|
57
|
+
def candidate(segments)
|
|
58
|
+
case segments
|
|
59
|
+
in [YEAR => year, DAY => month, DAY => day, _, *] then [year, month, day]
|
|
60
|
+
in [segment, *rest] if (match = JOINED.match(segment)) && (!match[5] || rest.any?)
|
|
61
|
+
match.values_at(1, 3, 4)
|
|
62
|
+
else
|
|
63
|
+
nil
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# A path with any implausible or second distinct date is ambiguous, so nil stays in the list.
|
|
68
|
+
def plausible(parts, latest)
|
|
69
|
+
return unless ::Date.valid_date?(*parts)
|
|
70
|
+
|
|
71
|
+
date = ::Date.new(*parts)
|
|
72
|
+
date if date.year >= MIN_YEAR && date <= latest
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -72,7 +72,7 @@ module Html2rss
|
|
|
72
72
|
def extract_url
|
|
73
73
|
@extract_url ||= begin
|
|
74
74
|
href = @selected_anchor.attrs.href.to_s
|
|
75
|
-
Url.
|
|
75
|
+
Url.resolve_web(href.split('#').first, @base_url, web_only: false)
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
78
|
|
|
@@ -211,7 +211,7 @@ module Html2rss
|
|
|
211
211
|
|
|
212
212
|
def extract_image
|
|
213
213
|
img_src = image_from_srcset || image_from_img || image_from_style || image_from_poster
|
|
214
|
-
Url.
|
|
214
|
+
Url.resolve_web(img_src, @base_url)
|
|
215
215
|
end
|
|
216
216
|
|
|
217
217
|
def image_from_img
|
|
@@ -41,18 +41,10 @@ module Html2rss
|
|
|
41
41
|
# @param url [Html2rss::Url]
|
|
42
42
|
# @return [PathClassifier] path classifier that also sees the first host label (an obituary subdomain)
|
|
43
43
|
def self.classifier_for(url)
|
|
44
|
-
PathClassifier.new(url.
|
|
44
|
+
PathClassifier.new(url.decoded_path_segments,
|
|
45
45
|
host_label: url.host.to_s.split('.').first, query: !url.query.nil?)
|
|
46
46
|
end
|
|
47
47
|
private_class_method :classifier_for
|
|
48
|
-
|
|
49
|
-
# Url keeps path segments percent-encoded; classification words are plain NFC text.
|
|
50
|
-
# @param segment [String] percent-encoded path segment
|
|
51
|
-
# @return [String] decoded, NFC-normalized segment
|
|
52
|
-
def self.decode_segment(segment)
|
|
53
|
-
Addressable::URI.unencode_component(segment).force_encoding(Encoding::UTF_8).scrub.unicode_normalize(:nfc)
|
|
54
|
-
end
|
|
55
|
-
private_class_method :decode_segment
|
|
56
48
|
end
|
|
57
49
|
end
|
|
58
50
|
end
|
|
@@ -105,7 +105,7 @@ module Html2rss
|
|
|
105
105
|
deep_post_context: Set['press', 'newsroom', 'presse', 'pressemitteilungen', 'prensa'].freeze
|
|
106
106
|
}.freeze
|
|
107
107
|
end
|
|
108
|
-
#
|
|
108
|
+
# Routing shape: any digit-led slug or id (4+ digits), not a calendar year; see ArticleRules::UrlDate::YEAR.
|
|
109
109
|
YEARISH_SEGMENT = /\A\d{4,}[\w-]*\z/
|
|
110
110
|
# Hyphenated slug shape common to article permalinks.
|
|
111
111
|
POST_SLUG_SEGMENT = /\A[a-z0-9]+(?:-[a-z0-9]+){2,}\z/i
|
|
@@ -113,7 +113,18 @@ module Html2rss
|
|
|
113
113
|
# Final label must be alphabetic (TLD-like); exclude common file extensions.
|
|
114
114
|
HOST_SHAPED_SEGMENT = /\A(?:www\.)?(?:[\w-]+\.)+[a-z]{2,24}\z/i
|
|
115
115
|
# Common file-extension suffixes excluded from host-shaped junk matching.
|
|
116
|
-
|
|
116
|
+
# Video files: a path ending in one is a media asset, never an article. Audio stays out (podcast items
|
|
117
|
+
# link their enclosure) and so do images (`/assets/image.jpg` is pinned as not junk).
|
|
118
|
+
MEDIA_FILE_EXTENSIONS = %w[mp4 webm flv mov m4v].freeze
|
|
119
|
+
# All file-extension suffixes, media included (single owner of the list).
|
|
120
|
+
FILE_EXTENSIONS = (%w[pdf jpe?g png gif svg webp css js mjs html? xml json zip gz] + MEDIA_FILE_EXTENSIONS).freeze
|
|
121
|
+
# Segment ending in a known file extension.
|
|
122
|
+
FILE_EXTENSION_SEGMENT = /\.(?:#{FILE_EXTENSIONS.join('|')})\z/i
|
|
123
|
+
# Segment ending in a video file extension.
|
|
124
|
+
MEDIA_FILE_SEGMENT = /\.(?:#{MEDIA_FILE_EXTENSIONS.join('|')})\z/i
|
|
125
|
+
# Document-style permalink: four or more words, then a numeric id of 5+ digits and a page extension
|
|
126
|
+
# (`/author/the-long-title_b_4421799.html`). A text-only slug is too common on product, forum and wiki pages.
|
|
127
|
+
DOCUMENT_SLUG_SEGMENT = /\A[\p{L}\d]+(?:[-_][\p{L}\d]+){3,}[-_]\d{5,}\.(?:html?|shtml|aspx?|php)\z/i
|
|
117
128
|
|
|
118
129
|
# A single person-name slug: letters only (dated and id slugs carry digits), up to three words.
|
|
119
130
|
NAME_SLUG = /\A\p{L}+(?:-\p{L}+){0,2}\z/
|
|
@@ -166,8 +177,8 @@ module Html2rss
|
|
|
166
177
|
# @return [Boolean] true when the final path segment looks like a post slug
|
|
167
178
|
def strong_post_suffix?
|
|
168
179
|
@strong_post_suffix ||= segments.any? &&
|
|
169
|
-
slug_last_segment? &&
|
|
170
|
-
|
|
180
|
+
((slug_last_segment? && trusted_post_context?(segments.size - 1)) ||
|
|
181
|
+
document_slug_route?)
|
|
171
182
|
end
|
|
172
183
|
|
|
173
184
|
# @return [Boolean] true when every path segment is utility chrome
|
|
@@ -190,7 +201,8 @@ module Html2rss
|
|
|
190
201
|
def non_article_path?
|
|
191
202
|
return @non_article_path if defined?(@non_article_path)
|
|
192
203
|
|
|
193
|
-
@non_article_path = section_index? || profile_route? ||
|
|
204
|
+
@non_article_path = section_index? || profile_route? || media_file_route? ||
|
|
205
|
+
(!strong_post_suffix? && non_article_word_present?)
|
|
194
206
|
end
|
|
195
207
|
|
|
196
208
|
# @return [Boolean] true when the route is shallow and contains high-confidence noise
|
|
@@ -220,6 +232,10 @@ module Html2rss
|
|
|
220
232
|
SEGMENT_SETS[:non_article].include?(joined) || joined.match?(PLAYER_SEGMENT)
|
|
221
233
|
end
|
|
222
234
|
|
|
235
|
+
def media_file_route?
|
|
236
|
+
segments.any? && segments.last.match?(MEDIA_FILE_SEGMENT)
|
|
237
|
+
end
|
|
238
|
+
|
|
223
239
|
# A people word that ends the path (an index) or is followed by one name slug (a profile page).
|
|
224
240
|
def profile_route?
|
|
225
241
|
return false if segments.empty?
|
|
@@ -290,6 +306,16 @@ module Html2rss
|
|
|
290
306
|
end
|
|
291
307
|
end
|
|
292
308
|
|
|
309
|
+
# A document slug needs no content/year context, unless a slug word is utility chrome or the path names a
|
|
310
|
+
# non-article destination (games, quiz, ...).
|
|
311
|
+
def document_slug_route?
|
|
312
|
+
return false if segments.size < 2 || leading_high_confidence_junk? || non_article_word_present?
|
|
313
|
+
|
|
314
|
+
last = segments.last
|
|
315
|
+
utility = SEGMENT_SETS.fetch(:utility) # vanity words are a subset of utility
|
|
316
|
+
last.match?(DOCUMENT_SLUG_SEGMENT) && last.split(/[-_.]/).none? { utility.include?(_1.downcase) }
|
|
317
|
+
end
|
|
318
|
+
|
|
293
319
|
def slug_last_segment?
|
|
294
320
|
last = segments.last
|
|
295
321
|
last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)
|
|
@@ -43,7 +43,7 @@ module Html2rss
|
|
|
43
43
|
|
|
44
44
|
# @return [HTTPX::Session]
|
|
45
45
|
def base_ssrf_session
|
|
46
|
-
@base_ssrf_session ||= base_session.plugin(
|
|
46
|
+
@base_ssrf_session ||= base_session.plugin(NetworkGuardFilter)
|
|
47
47
|
end
|
|
48
48
|
# rubocop:enable ThreadSafety/ClassInstanceVariable
|
|
49
49
|
end
|
|
@@ -27,8 +27,17 @@ module Html2rss
|
|
|
27
27
|
IPAddr.new('::1/128'),
|
|
28
28
|
IPAddr.new('fe80::/10'),
|
|
29
29
|
IPAddr.new('fc00::/7'),
|
|
30
|
-
IPAddr.new('ff00::/8')
|
|
30
|
+
IPAddr.new('ff00::/8'),
|
|
31
|
+
# RFC 8215 local-use NAT64 prefix: not globally reachable.
|
|
32
|
+
IPAddr.new('64:ff9b:1::/48'),
|
|
33
|
+
# Teredo (RFC 4380): client address is obfuscated, so the whole prefix is denied.
|
|
34
|
+
IPAddr.new('2001::/32')
|
|
31
35
|
].freeze
|
|
36
|
+
# RFC 6052 well-known NAT64 prefix; the last 32 bits embed an IPv4 address.
|
|
37
|
+
NAT64_PREFIX = IPAddr.new('64:ff9b::/96')
|
|
38
|
+
# 6to4 (RFC 3056); bits 16..47 embed an IPv4 address. Decoded only to deny private embeds;
|
|
39
|
+
# the HTTPX filter still denies all of 2002::/16 at connect.
|
|
40
|
+
SIX_TO_FOUR_PREFIX = IPAddr.new('2002::/16')
|
|
32
41
|
|
|
33
42
|
##
|
|
34
43
|
# @param allow_private_networks [Boolean] whether private network targets are allowed
|
|
@@ -75,6 +84,31 @@ module Html2rss
|
|
|
75
84
|
raise PrivateNetworkDenied, "Private network target denied for #{url}"
|
|
76
85
|
end
|
|
77
86
|
|
|
87
|
+
##
|
|
88
|
+
# Single owner of the IP allow/deny decision. IPv6 forms that embed an IPv4 address
|
|
89
|
+
# (IPv4-mapped, NAT64 64:ff9b::/96, 6to4) are allowed only if the embedded IPv4 is.
|
|
90
|
+
#
|
|
91
|
+
# @param address [IPAddr] resolved or remote address
|
|
92
|
+
# @return [Boolean] true when the address must be denied
|
|
93
|
+
def blocked_ip?(address)
|
|
94
|
+
native = address.native
|
|
95
|
+
return true if BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
|
|
96
|
+
|
|
97
|
+
embedded = embedded_ipv4(native)
|
|
98
|
+
!embedded.nil? && blocked_ip?(embedded)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
##
|
|
102
|
+
# @param address [IPAddr] native (non-mapped) address
|
|
103
|
+
# @return [IPAddr, nil] the IPv4 embedded in a NAT64 or 6to4 address, else nil
|
|
104
|
+
def embedded_ipv4(address)
|
|
105
|
+
if NAT64_PREFIX.include?(address)
|
|
106
|
+
IPAddr.new_ntoh(address.hton[12..])
|
|
107
|
+
elsif SIX_TO_FOUR_PREFIX.include?(address)
|
|
108
|
+
IPAddr.new_ntoh(address.hton[2, 4])
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
78
112
|
private
|
|
79
113
|
|
|
80
114
|
attr_reader :resolver
|
|
@@ -120,11 +154,6 @@ module Html2rss
|
|
|
120
154
|
rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
|
|
121
155
|
nil
|
|
122
156
|
end
|
|
123
|
-
|
|
124
|
-
def blocked_ip?(address)
|
|
125
|
-
native = address.native
|
|
126
|
-
BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
|
|
127
|
-
end
|
|
128
157
|
end
|
|
129
158
|
end
|
|
130
159
|
end
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'httpx'
|
|
4
|
+
require 'httpx/plugins/ssrf_filter'
|
|
5
|
+
|
|
6
|
+
module Html2rss
|
|
7
|
+
class RequestService
|
|
8
|
+
##
|
|
9
|
+
# HTTPX plugin layered on +:ssrf_filter+ (HTTPX 1.9.x, which offers no per-address predicate).
|
|
10
|
+
# +ssrf_filter+ denies the whole NAT64 prefix; here it is released via +safe_private_ranges+
|
|
11
|
+
# only together with a per-address check by {NetworkGuard} (embedded IPv4) and HTTPX's own
|
|
12
|
+
# IPv4 list on the embedded address. Either denying an address drops it (fail closed).
|
|
13
|
+
# Loading +:ssrf_filter+ first as a dependency keeps this module ahead of it in the
|
|
14
|
+
# ancestors; a spec pins that order.
|
|
15
|
+
#
|
|
16
|
+
# The release is gated at load: it is only applied on the tested HTTPX series and while the
|
|
17
|
+
# +ssrf_filter+ connection still routes resolution through +addresses=+. Otherwise HTTPX keeps
|
|
18
|
+
# denying all of 64:ff9b::/96 (fail closed) and a warning says NAT64 support is inactive.
|
|
19
|
+
#
|
|
20
|
+
# 6to4 is decoded by {NetworkGuard} only to deny private embedded IPv4; HTTPX itself still
|
|
21
|
+
# denies all of 2002::/16 at connect time.
|
|
22
|
+
module NetworkGuardFilter
|
|
23
|
+
# Strict guard instance: the plugin only exists on the SSRF-protected session.
|
|
24
|
+
GUARD = NetworkGuard.new(allow_private_networks: false)
|
|
25
|
+
|
|
26
|
+
# HTTPX series the release was verified against.
|
|
27
|
+
TESTED_HTTPX_SERIES = '1.9.'
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
# @param klass [Class] HTTPX session class being extended
|
|
31
|
+
# @return [void]
|
|
32
|
+
def load_dependencies(klass)
|
|
33
|
+
klass.plugin(:ssrf_filter)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @param options [HTTPX::Options] session options
|
|
37
|
+
# @return [HTTPX::Options] options with the NAT64 prefix released to {NetworkGuard}
|
|
38
|
+
# when HTTPX is in the tested range; unchanged otherwise
|
|
39
|
+
def extra_options(options)
|
|
40
|
+
return options.merge(safe_private_ranges: [NetworkGuard::NAT64_PREFIX]) if release_supported?
|
|
41
|
+
|
|
42
|
+
warn_inactive
|
|
43
|
+
options
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [Boolean] true when the installed HTTPX is the tested series and still hooks +addresses=+
|
|
47
|
+
def release_supported?
|
|
48
|
+
HTTPX::VERSION.start_with?(TESTED_HTTPX_SERIES) &&
|
|
49
|
+
HTTPX::Plugins::SsrfFilter::ConnectionMethods.method_defined?(:addresses=)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Logs once per process; carries no addresses.
|
|
53
|
+
# @return [void]
|
|
54
|
+
def warn_inactive
|
|
55
|
+
# rubocop:disable ThreadSafety/ClassInstanceVariable -- benign once-only log flag
|
|
56
|
+
return if @warned_inactive
|
|
57
|
+
|
|
58
|
+
@warned_inactive = true
|
|
59
|
+
# rubocop:enable ThreadSafety/ClassInstanceVariable
|
|
60
|
+
Log.warn({ event_name: 'network_guard.nat64_inactive', httpx_version: HTTPX::VERSION }.to_json)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @param ipaddr [IPAddr]
|
|
64
|
+
# @return [Boolean] true when the guard or HTTPX's IPv4 list denies the address
|
|
65
|
+
def denied?(ipaddr)
|
|
66
|
+
return true if GUARD.blocked_ip?(ipaddr)
|
|
67
|
+
|
|
68
|
+
embedded = GUARD.embedded_ipv4(ipaddr)
|
|
69
|
+
!embedded.nil? && HTTPX::Plugins::SsrfFilter.unsafe_ip_address?(embedded)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Connection hooks.
|
|
74
|
+
module ConnectionMethods
|
|
75
|
+
# @param addrs [Array<#address>] resolved entries
|
|
76
|
+
# @return [void]
|
|
77
|
+
# @raise [HTTPX::ServerSideRequestForgeryError] when no entry is allowed
|
|
78
|
+
def addresses=(addrs)
|
|
79
|
+
super(addrs.reject { |entry| NetworkGuardFilter.denied?(entry.address) })
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -19,9 +19,7 @@ module Html2rss
|
|
|
19
19
|
return nil unless node
|
|
20
20
|
|
|
21
21
|
href = node['href'] || node.text
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
Html2rss::Url.from_relative(href, page_response.url)
|
|
22
|
+
Html2rss::Url.resolve_web(href, page_response.url)
|
|
25
23
|
end
|
|
26
24
|
|
|
27
25
|
# @param parsed_body [Nokogiri::HTML::Document]
|
|
@@ -32,7 +32,7 @@ module Html2rss
|
|
|
32
32
|
return nil unless (next_url_path = config[:next_url_path])
|
|
33
33
|
|
|
34
34
|
next_url = dig_path(json_data, next_url_path)
|
|
35
|
-
Html2rss::Url.
|
|
35
|
+
Html2rss::Url.resolve_web(next_url, origin_url)
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# @param json_data [Hash]
|
|
@@ -13,9 +13,7 @@ module Html2rss
|
|
|
13
13
|
# @return [String, nil]
|
|
14
14
|
def next_page_url(page_response, page_number:) # rubocop:disable Lint/UnusedMethodArgument
|
|
15
15
|
href = page_response.parsed_body.at_css('link[rel~="next"][href], a[rel~="next"][href]')&.[]('href')
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
Html2rss::Url.from_relative(href, page_response.url)
|
|
16
|
+
Html2rss::Url.resolve_web(href, page_response.url)
|
|
19
17
|
end
|
|
20
18
|
end
|
|
21
19
|
end
|
|
@@ -59,6 +59,17 @@ module Html2rss
|
|
|
59
59
|
Pager.for(pagination_config, session: self, initial_response:)
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
+
##
|
|
63
|
+
# Returns a session with the same strategy, policy, headers and origin but its own request budget, so a
|
|
64
|
+
# bounded optional step (article enrichment) cannot spend the listing's request slots. The wall clock is
|
|
65
|
+
# the remaining time of this session's budget (rounded up, at least one second).
|
|
66
|
+
#
|
|
67
|
+
# @param max_requests [Integer] request slots of the new budget
|
|
68
|
+
# @return [RequestSession]
|
|
69
|
+
def with_budget_allowance(max_requests)
|
|
70
|
+
self.class.new(context: allowance_context(max_requests), strategy:, logger:)
|
|
71
|
+
end
|
|
72
|
+
|
|
62
73
|
##
|
|
63
74
|
# Executes a follow-up request sharing policy, headers, and budget.
|
|
64
75
|
#
|
|
@@ -123,6 +134,15 @@ module Html2rss
|
|
|
123
134
|
|
|
124
135
|
attr_reader :context, :strategy, :logger, :visited_urls
|
|
125
136
|
|
|
137
|
+
def allowance_context(max_requests)
|
|
138
|
+
remaining = context.budget.remaining_timeout_seconds
|
|
139
|
+
budget = RequestService::Budget.new(max_requests:, total_timeout_seconds: remaining && [remaining.ceil, 1].max)
|
|
140
|
+
RequestService::Context.new(
|
|
141
|
+
url:, headers: context.headers, request: context.request, policy: context.policy, budget:,
|
|
142
|
+
origin_url: context.origin_url
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
|
|
126
146
|
def execute(request_context)
|
|
127
147
|
RequestService.execute(request_context, strategy:).tap do |response|
|
|
128
148
|
logger.debug(
|
|
@@ -54,9 +54,9 @@ module Html2rss
|
|
|
54
54
|
##
|
|
55
55
|
# Retrieves and returns the normalized absolute URL.
|
|
56
56
|
#
|
|
57
|
-
# @return [Html2rss::Url, nil] The absolute URL, or nil when nothing matched
|
|
57
|
+
# @return [Html2rss::Url, nil] The absolute URL, or nil when nothing matched the +href+ is blank or unparseable.
|
|
58
58
|
def call
|
|
59
|
-
Url.
|
|
59
|
+
Url.resolve_web(@href, @base_url, web_only: false)
|
|
60
60
|
end
|
|
61
61
|
end
|
|
62
62
|
end
|
data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
CHANGED
|
@@ -33,7 +33,8 @@ module Html2rss
|
|
|
33
33
|
|
|
34
34
|
url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[node_name]
|
|
35
35
|
url = node[url_attribute]
|
|
36
|
-
|
|
36
|
+
resolved = Url.resolve_web(url, @base_url, web_only: false)
|
|
37
|
+
node[url_attribute] = resolved.to_s if resolved
|
|
37
38
|
end
|
|
38
39
|
end
|
|
39
40
|
end
|
|
@@ -48,9 +48,9 @@ module Html2rss
|
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
##
|
|
51
|
-
# @return [String]
|
|
51
|
+
# @return [String, nil] nil when +value+ is blank or unparseable
|
|
52
52
|
def call
|
|
53
|
-
Url.
|
|
53
|
+
Url.resolve_web(value, context.base_url, web_only: false)&.to_s
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
56
|
end
|
data/lib/html2rss/selectors.rb
CHANGED
|
@@ -452,10 +452,7 @@ module Html2rss
|
|
|
452
452
|
end
|
|
453
453
|
|
|
454
454
|
def default_item_url(item, base_url = @url)
|
|
455
|
-
|
|
456
|
-
return if href.empty?
|
|
457
|
-
|
|
458
|
-
Url.from_relative(href, base_url)
|
|
455
|
+
Url.resolve_web(item['href'], base_url, web_only: false)
|
|
459
456
|
end
|
|
460
457
|
|
|
461
458
|
def item_env_for(item, base_url)
|
data/lib/html2rss/status.rb
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
Status = Data.define(
|
|
5
5
|
:version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count,
|
|
6
|
-
:strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution
|
|
6
|
+
:strategy_attempts, :admission_drops, :entry_url, :scrape_url, :entry_resolution,
|
|
7
|
+
:enriched, :enrich_failed, :enrich_skipped_cross_origin
|
|
7
8
|
)
|
|
8
9
|
|
|
9
10
|
##
|
|
@@ -12,7 +13,7 @@ module Html2rss
|
|
|
12
13
|
# Exposed publicly via {FeedResult#status}. Safe to log without reading articles.
|
|
13
14
|
# Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
|
|
14
15
|
# Tallies and counters are validated and frozen at construction (including Marshal load).
|
|
15
|
-
class Status
|
|
16
|
+
class Status # rubocop:disable Metrics/ClassLength -- one reader and one validated counter per telemetry field
|
|
16
17
|
# Raised when telemetry fields fail validation at construction.
|
|
17
18
|
class InvalidError < Html2rss::ContractError; end
|
|
18
19
|
|
|
@@ -20,6 +21,10 @@ module Html2rss
|
|
|
20
21
|
ALWAYS_REPORTED = %i[version dedup_dropped].freeze
|
|
21
22
|
private_constant :ALWAYS_REPORTED
|
|
22
23
|
|
|
24
|
+
# Article-enrichment counters (see {Article::Enrichment::Result}).
|
|
25
|
+
ENRICHMENT_KEYS = %i[enriched enrich_failed enrich_skipped_cross_origin].freeze
|
|
26
|
+
private_constant :ENRICHMENT_KEYS
|
|
27
|
+
|
|
23
28
|
class << self
|
|
24
29
|
##
|
|
25
30
|
# Builds status from extracted articles and scrape telemetry.
|
|
@@ -34,11 +39,12 @@ module Html2rss
|
|
|
34
39
|
# @param entry_url [String, nil] original channel URL when entry resolution ran
|
|
35
40
|
# @param scrape_url [String, nil] effective scrape URL after resolution
|
|
36
41
|
# @param entry_resolution [Html2rss::FeedResolution::Diag, Hash, nil] resolution diagnostics
|
|
42
|
+
# @param enrichment [Hash{Symbol => Integer}] {Article::Enrichment} tally (empty when off)
|
|
37
43
|
# @return [Html2rss::Status]
|
|
38
44
|
# rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Status kwargs stay co-located
|
|
39
45
|
def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0,
|
|
40
46
|
strategy_attempts: [], admission_drops: {}, scrape_target: nil,
|
|
41
|
-
entry_url: nil, scrape_url: nil, entry_resolution: nil)
|
|
47
|
+
entry_url: nil, scrape_url: nil, entry_resolution: nil, enrichment: {})
|
|
42
48
|
tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
|
|
43
49
|
new(
|
|
44
50
|
version: Html2rss::VERSION,
|
|
@@ -50,7 +56,8 @@ module Html2rss
|
|
|
50
56
|
admission_drops:,
|
|
51
57
|
entry_url: entry_url || scrape_target&.entry_url,
|
|
52
58
|
scrape_url: scrape_url || scrape_target&.effective_url,
|
|
53
|
-
entry_resolution
|
|
59
|
+
entry_resolution:,
|
|
60
|
+
**ENRICHMENT_KEYS.to_h { [_1, enrichment.fetch(_1, 0)] }
|
|
54
61
|
)
|
|
55
62
|
end
|
|
56
63
|
|
|
@@ -73,10 +80,14 @@ module Html2rss
|
|
|
73
80
|
# @param entry_url [String, nil]
|
|
74
81
|
# @param scrape_url [String, nil]
|
|
75
82
|
# @param entry_resolution [Html2rss::FeedResolution::Diag, Hash, nil]
|
|
83
|
+
# @param enriched [Integer] items that gained a field from their article page
|
|
84
|
+
# @param enrich_failed [Integer] article fetches that failed or were blocked
|
|
85
|
+
# @param enrich_skipped_cross_origin [Integer] fetches skipped by the same-origin policy (subset of enrich_failed)
|
|
76
86
|
# rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
|
|
77
87
|
def initialize(
|
|
78
88
|
version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
|
|
79
|
-
strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil
|
|
89
|
+
strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil,
|
|
90
|
+
enriched: 0, enrich_failed: 0, enrich_skipped_cross_origin: 0
|
|
80
91
|
)
|
|
81
92
|
dedup = Option.integer(:dedup_dropped, dedup_dropped, min: 0)
|
|
82
93
|
attempts = Option.integer(:attempt_count, attempt_count, min: 0)
|
|
@@ -92,7 +103,10 @@ module Html2rss
|
|
|
92
103
|
admission_drops: freeze_tallies(admission_drops),
|
|
93
104
|
entry_url: freeze_optional_string(entry_url),
|
|
94
105
|
scrape_url: freeze_optional_string(scrape_url),
|
|
95
|
-
entry_resolution: freeze_entry_resolution(entry_resolution)
|
|
106
|
+
entry_resolution: freeze_entry_resolution(entry_resolution),
|
|
107
|
+
enriched: Option.integer(:enriched, enriched, min: 0),
|
|
108
|
+
enrich_failed: Option.integer(:enrich_failed, enrich_failed, min: 0),
|
|
109
|
+
enrich_skipped_cross_origin: Option.integer(:enrich_skipped_cross_origin, enrich_skipped_cross_origin, min: 0)
|
|
96
110
|
)
|
|
97
111
|
end
|
|
98
112
|
|
|
@@ -129,15 +143,18 @@ module Html2rss
|
|
|
129
143
|
|
|
130
144
|
def marshal_dump
|
|
131
145
|
[version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
|
|
132
|
-
admission_drops, entry_url, scrape_url, entry_resolution
|
|
146
|
+
admission_drops, entry_url, scrape_url, entry_resolution, enriched, enrich_failed,
|
|
147
|
+
enrich_skipped_cross_origin]
|
|
133
148
|
end
|
|
134
149
|
|
|
135
150
|
def marshal_load(payload)
|
|
136
151
|
version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
|
|
137
|
-
strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution
|
|
152
|
+
strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution,
|
|
153
|
+
enriched, enrich_failed, enrich_skipped_cross_origin = payload
|
|
138
154
|
initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
|
|
139
155
|
strategy_attempts:, admission_drops:, entry_url:, scrape_url:,
|
|
140
|
-
entry_resolution:
|
|
156
|
+
entry_resolution:, enriched: enriched || 0, enrich_failed: enrich_failed || 0,
|
|
157
|
+
enrich_skipped_cross_origin: enrich_skipped_cross_origin || 0)
|
|
141
158
|
end
|
|
142
159
|
|
|
143
160
|
def freeze_tallies(tallies)
|
data/lib/html2rss/url.rb
CHANGED
|
@@ -45,6 +45,15 @@ module Html2rss
|
|
|
45
45
|
SCHEME_PREFIX = /\A([a-z][a-z0-9+.-]*):/i
|
|
46
46
|
private_constant :SCHEME_PREFIX
|
|
47
47
|
|
|
48
|
+
# One DNS label: 1-63 letters, digits, underscores and inner hyphens (IDN labels arrive punycoded).
|
|
49
|
+
DNS_LABEL = '[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?'
|
|
50
|
+
DNS_NAME = /\A(?:#{DNS_LABEL}\.)*#{DNS_LABEL}\z/i
|
|
51
|
+
OCTET = '(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)'
|
|
52
|
+
IPV4 = /\A(?:#{OCTET}\.){3}#{OCTET}\z/
|
|
53
|
+
NUMERIC_LAST_LABEL = /(?:\A|\.)\d+\z/
|
|
54
|
+
MAX_HOST_LENGTH = 253
|
|
55
|
+
private_constant :DNS_LABEL, :DNS_NAME, :OCTET, :IPV4, :NUMERIC_LAST_LABEL, :MAX_HOST_LENGTH
|
|
56
|
+
|
|
48
57
|
##
|
|
49
58
|
# Creates a URL from a relative path and base URL.
|
|
50
59
|
#
|
|
@@ -105,6 +114,8 @@ module Html2rss
|
|
|
105
114
|
return unless match
|
|
106
115
|
|
|
107
116
|
new(parse(match[0]).normalize)
|
|
117
|
+
rescue Addressable::URI::InvalidURIError, InvalidUrlError
|
|
118
|
+
nil
|
|
108
119
|
end
|
|
109
120
|
|
|
110
121
|
##
|
|
@@ -160,12 +171,13 @@ module Html2rss
|
|
|
160
171
|
#
|
|
161
172
|
# @param ref [String, Html2rss::Url, nil] an absolute or relative URL, possibly padded with whitespace
|
|
162
173
|
# @param base_url [String, Html2rss::Url, nil] the document the reference appears in
|
|
163
|
-
# @
|
|
164
|
-
|
|
174
|
+
# @param web_only [Boolean] keep only http(s); +false+ also keeps other schemes (+mailto:+ and the like)
|
|
175
|
+
# @return [Url, nil] the absolute URL, or nil when +ref+ is blank, unparseable or (with +web_only+) not http(s)
|
|
176
|
+
def self.resolve_web(ref, base_url = nil, web_only: true)
|
|
165
177
|
return unless ref.to_s.match?(/\S/)
|
|
166
178
|
|
|
167
179
|
url = from_relative(ref, base_url)
|
|
168
|
-
url if url.web?
|
|
180
|
+
url if !web_only || url.web?
|
|
169
181
|
rescue InvalidUrlError
|
|
170
182
|
nil
|
|
171
183
|
end
|
|
@@ -213,6 +225,8 @@ module Html2rss
|
|
|
213
225
|
#
|
|
214
226
|
# @param uri [Addressable::URI] the normalized Addressable::URI (internal use only)
|
|
215
227
|
def initialize(uri)
|
|
228
|
+
raise InvalidUrlError, 'URL host is invalid' unless valid_host?(uri.host)
|
|
229
|
+
|
|
216
230
|
@uri = uri
|
|
217
231
|
@path_segments = path.split('/').reject(&:empty?).freeze
|
|
218
232
|
freeze
|
|
@@ -271,6 +285,14 @@ module Html2rss
|
|
|
271
285
|
# @return [Array<String>] normalized path segments
|
|
272
286
|
attr_reader :path_segments
|
|
273
287
|
|
|
288
|
+
##
|
|
289
|
+
# Path segments as plain text: percent-decoded, scrubbed and NFC-normalized (for matching words and digits).
|
|
290
|
+
#
|
|
291
|
+
# @return [Array<String>] decoded path segments
|
|
292
|
+
def decoded_path_segments
|
|
293
|
+
path_segments.map { Addressable::URI.unencode_component(_1).force_encoding(Encoding::UTF_8).scrub.unicode_normalize(:nfc) }
|
|
294
|
+
end
|
|
295
|
+
|
|
274
296
|
##
|
|
275
297
|
# Returns a copy of the URL with the provided path.
|
|
276
298
|
#
|
|
@@ -440,6 +462,35 @@ module Html2rss
|
|
|
440
462
|
|
|
441
463
|
def tracking_param?(pair) = TRACKING_PARAM.match?(pair.partition('=').first)
|
|
442
464
|
|
|
465
|
+
##
|
|
466
|
+
# The one host rule: no host (relative or opaque URL), a DNS name, an IPv4 literal, or a bracketed IPv6
|
|
467
|
+
# literal that parses. Addressable's +normalize+ already strips a trailing dot; a Unicode host is checked in
|
|
468
|
+
# its punycode form. An IPv4 octet has no leading zero (+01.2.3.4+, +0177.0.0.1+): readers disagree on
|
|
469
|
+
# whether that is decimal or octal, the classic shape of an address-guard bypass.
|
|
470
|
+
#
|
|
471
|
+
# @param host [String, nil]
|
|
472
|
+
# @return [Boolean] truthy when valid
|
|
473
|
+
def valid_host?(host)
|
|
474
|
+
return true if host.nil? || host.empty?
|
|
475
|
+
|
|
476
|
+
host = Addressable::IDNA.to_ascii(host) unless host.ascii_only?
|
|
477
|
+
return ipv6_literal?(host) if host.start_with?('[')
|
|
478
|
+
|
|
479
|
+
dns_name_or_ipv4?(host)
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
def ipv6_literal?(host)
|
|
483
|
+
IPAddr.new(host, Socket::AF_INET6)
|
|
484
|
+
rescue IPAddr::Error
|
|
485
|
+
false
|
|
486
|
+
end
|
|
487
|
+
|
|
488
|
+
def dns_name_or_ipv4?(name)
|
|
489
|
+
return IPV4.match?(name) if NUMERIC_LAST_LABEL.match?(name)
|
|
490
|
+
|
|
491
|
+
name.length <= MAX_HOST_LENGTH && DNS_NAME.match?(name)
|
|
492
|
+
end
|
|
493
|
+
|
|
443
494
|
def ip_address_host?
|
|
444
495
|
!IPAddr.new(host).nil?
|
|
445
496
|
rescue IPAddr::Error
|
data/lib/html2rss/version.rb
CHANGED
|
@@ -855,6 +855,26 @@
|
|
|
855
855
|
}
|
|
856
856
|
},
|
|
857
857
|
"required": []
|
|
858
|
+
},
|
|
859
|
+
"enrich": {
|
|
860
|
+
"type": "object",
|
|
861
|
+
"properties": {
|
|
862
|
+
"article_metadata": {
|
|
863
|
+
"type": "boolean",
|
|
864
|
+
"not": {
|
|
865
|
+
"type": "null"
|
|
866
|
+
}
|
|
867
|
+
},
|
|
868
|
+
"max_fetches": {
|
|
869
|
+
"type": "integer",
|
|
870
|
+
"not": {
|
|
871
|
+
"type": "null"
|
|
872
|
+
},
|
|
873
|
+
"exclusiveMinimum": 0,
|
|
874
|
+
"maximum": 10
|
|
875
|
+
}
|
|
876
|
+
},
|
|
877
|
+
"required": []
|
|
858
878
|
}
|
|
859
879
|
},
|
|
860
880
|
"required": [
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: html2rss
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.35.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Gil Desmarais
|
|
@@ -270,6 +270,7 @@ files:
|
|
|
270
270
|
- lib/html2rss/article.rb
|
|
271
271
|
- lib/html2rss/article/deduplicator.rb
|
|
272
272
|
- lib/html2rss/article/enclosure.rb
|
|
273
|
+
- lib/html2rss/article/enrichment.rb
|
|
273
274
|
- lib/html2rss/article/title.rb
|
|
274
275
|
- lib/html2rss/auto_source.rb
|
|
275
276
|
- lib/html2rss/auto_source/README.md
|
|
@@ -372,6 +373,7 @@ files:
|
|
|
372
373
|
- lib/html2rss/html/article_rules/description.rb
|
|
373
374
|
- lib/html2rss/html/article_rules/enclosure.rb
|
|
374
375
|
- lib/html2rss/html/article_rules/image.rb
|
|
376
|
+
- lib/html2rss/html/article_rules/url_date.rb
|
|
375
377
|
- lib/html2rss/html/card_walk.rb
|
|
376
378
|
- lib/html2rss/html/feed_link.rb
|
|
377
379
|
- lib/html2rss/html/navigator.rb
|
|
@@ -424,6 +426,7 @@ files:
|
|
|
424
426
|
- lib/html2rss/request_service/httpx_strategy.rb
|
|
425
427
|
- lib/html2rss/request_service/local_file_strategy.rb
|
|
426
428
|
- lib/html2rss/request_service/network_guard.rb
|
|
429
|
+
- lib/html2rss/request_service/network_guard_filter.rb
|
|
427
430
|
- lib/html2rss/request_service/policy.rb
|
|
428
431
|
- lib/html2rss/request_service/replay_breaker.rb
|
|
429
432
|
- lib/html2rss/request_service/response.rb
|
|
@@ -507,7 +510,7 @@ licenses:
|
|
|
507
510
|
- MIT
|
|
508
511
|
metadata:
|
|
509
512
|
allowed_push_host: https://rubygems.org
|
|
510
|
-
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.
|
|
513
|
+
changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.35.0
|
|
511
514
|
rubygems_mfa_required: 'true'
|
|
512
515
|
rdoc_options: []
|
|
513
516
|
require_paths:
|