html2rss 0.32.3 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/html2rss/article/enclosure.rb +2 -6
- data/lib/html2rss/auto_source/cleanup.rb +1 -3
- data/lib/html2rss/auto_source/scraper/html.rb +2 -8
- data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
- data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
- data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
- data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
- data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
- data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
- data/lib/html2rss/auto_source/scraper.rb +0 -11
- data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
- data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
- data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
- data/lib/html2rss/auto_source.rb +1 -1
- data/lib/html2rss/batch.rb +22 -32
- data/lib/html2rss/capture/README.md +1 -1
- data/lib/html2rss/capture.rb +7 -27
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +33 -27
- data/lib/html2rss/config/request_controls.rb +10 -2
- data/lib/html2rss/config/schema.rb +8 -10
- data/lib/html2rss/config/selectors_validator.rb +1 -0
- data/lib/html2rss/config/validator.rb +3 -3
- data/lib/html2rss/config.rb +1 -1
- data/lib/html2rss/defaults.rb +5 -16
- data/lib/html2rss/doctor/botasaurus.rb +11 -19
- data/lib/html2rss/error.rb +54 -1
- data/lib/html2rss/feed_builder/rss.rb +1 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +58 -55
- data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +3 -3
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +5 -9
- data/lib/html2rss/feed_result.rb +4 -3
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
- data/lib/html2rss/html/probe.rb +51 -51
- data/lib/html2rss/link_destination/path_classifier.rb +11 -33
- data/lib/html2rss/mcp/config_argument.rb +9 -2
- data/lib/html2rss/mcp/contract.rb +7 -5
- data/lib/html2rss/mcp/outcome.rb +4 -5
- data/lib/html2rss/mcp/runtime.rb +17 -24
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/diagnostics.rb +7 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +10 -12
- data/lib/html2rss/request_service/blocked_surface.rb +3 -7
- data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +1 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +3 -1
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +8 -26
- data/lib/html2rss/request_service/response.rb +3 -6
- data/lib/html2rss/request_service/strategy.rb +2 -2
- data/lib/html2rss/request_service.rb +26 -2
- data/lib/html2rss/request_session/pager/base.rb +3 -3
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/scoring/engine.rb +4 -32
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +14 -18
- data/lib/html2rss/sst/node.rb +9 -3
- data/lib/html2rss/sst/normalizer.rb +25 -24
- data/lib/html2rss/sst/text.rb +53 -57
- data/lib/html2rss/status.rb +23 -30
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/discovery.rb +117 -187
- data/lib/html2rss/syndication/parser.rb +92 -115
- data/lib/html2rss/test/enhance_audit.rb +13 -15
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +29 -48
- data/lib/html2rss/url.rb +53 -53
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +2 -1
- metadata +4 -2
data/lib/html2rss/mcp/runtime.rb
CHANGED
|
@@ -14,31 +14,24 @@ module Html2rss
|
|
|
14
14
|
:botasaurus_configured
|
|
15
15
|
)
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
class << self
|
|
18
|
+
##
|
|
19
|
+
# @return [Boolean] whether Botasaurus transport is configured in this process
|
|
20
|
+
def botasaurus_configured?
|
|
21
|
+
!ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
|
|
22
|
+
end
|
|
18
23
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
|
|
31
|
-
catalog_fingerprint: Contract.catalog_fingerprint,
|
|
32
|
-
tools: Contract.catalog_tools,
|
|
33
|
-
botasaurus_configured: botasaurus_configured?
|
|
34
|
-
)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
##
|
|
38
|
-
# @param strategy [String, Symbol, nil]
|
|
39
|
-
# @return [Symbol]
|
|
40
|
-
def coerce_strategy(strategy)
|
|
41
|
-
(strategy || :auto).to_sym
|
|
24
|
+
##
|
|
25
|
+
# @return [Snapshot] runtime capabilities and catalog identity for MCP clients
|
|
26
|
+
def snapshot
|
|
27
|
+
Snapshot.new(
|
|
28
|
+
version: Html2rss::VERSION,
|
|
29
|
+
mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
|
|
30
|
+
catalog_fingerprint: Contract.catalog_fingerprint,
|
|
31
|
+
tools: Contract.catalog_tools,
|
|
32
|
+
botasaurus_configured: botasaurus_configured?
|
|
33
|
+
)
|
|
34
|
+
end
|
|
42
35
|
end
|
|
43
36
|
end
|
|
44
37
|
end
|
|
@@ -36,9 +36,9 @@ module Html2rss
|
|
|
36
36
|
input_schema: Contract::INSPECT_INPUT_SCHEMA,
|
|
37
37
|
annotations: Contract::ANNOTATIONS_OPEN_WORLD,
|
|
38
38
|
call: lambda { |url:, strategy: 'auto', **|
|
|
39
|
-
Outcome.
|
|
39
|
+
Outcome.inspect_report(
|
|
40
40
|
report: PageRecon::Diagnostics.call(
|
|
41
|
-
url:, strategy:
|
|
41
|
+
url:, strategy: Option.strategy(strategy), deep: false
|
|
42
42
|
)
|
|
43
43
|
)
|
|
44
44
|
}
|
|
@@ -51,7 +51,7 @@ module Html2rss
|
|
|
51
51
|
input_schema: Contract::RECON_INPUT_SCHEMA,
|
|
52
52
|
annotations: Contract::ANNOTATIONS_OPEN_WORLD,
|
|
53
53
|
call: lambda { |url:, strategy: 'auto', **|
|
|
54
|
-
Outcome.recon(result: Html2rss.recon(url, strategy:
|
|
54
|
+
Outcome.recon(result: Html2rss.recon(url, strategy: Option.strategy(strategy)))
|
|
55
55
|
}
|
|
56
56
|
),
|
|
57
57
|
Tool.new(
|
|
@@ -63,7 +63,7 @@ module Html2rss
|
|
|
63
63
|
annotations: Contract::ANNOTATIONS_OPEN_WORLD,
|
|
64
64
|
call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, limit: 10, **|
|
|
65
65
|
Outcome.batch_scrape(
|
|
66
|
-
Batch.batch_scrape(urls:, strategy:
|
|
66
|
+
Batch.batch_scrape(urls:, strategy: Option.strategy(strategy), concurrency:, limit:)
|
|
67
67
|
)
|
|
68
68
|
}
|
|
69
69
|
),
|
|
@@ -76,7 +76,7 @@ module Html2rss
|
|
|
76
76
|
annotations: Contract::ANNOTATIONS_OPEN_WORLD,
|
|
77
77
|
call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, **|
|
|
78
78
|
Outcome.batch_inspect(
|
|
79
|
-
Batch.batch_inspect(urls:, strategy:
|
|
79
|
+
Batch.batch_inspect(urls:, strategy: Option.strategy(strategy), concurrency:)
|
|
80
80
|
)
|
|
81
81
|
}
|
|
82
82
|
),
|
|
@@ -89,7 +89,7 @@ module Html2rss
|
|
|
89
89
|
annotations: Contract::ANNOTATIONS_OPEN_WORLD,
|
|
90
90
|
call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, **|
|
|
91
91
|
Outcome.batch_recon(
|
|
92
|
-
Batch.batch_recon(urls:, strategy:
|
|
92
|
+
Batch.batch_recon(urls:, strategy: Option.strategy(strategy), concurrency:)
|
|
93
93
|
)
|
|
94
94
|
}
|
|
95
95
|
),
|
|
@@ -136,7 +136,7 @@ module Html2rss
|
|
|
136
136
|
compare_enhance: false, **kwargs|
|
|
137
137
|
feed_config = ConfigArgument.parse(config:, yaml:).config
|
|
138
138
|
test_args = { min_items:, strict_quality:, compare_enhance: }
|
|
139
|
-
test_args[:strategy] =
|
|
139
|
+
test_args[:strategy] = Option.strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
|
|
140
140
|
Outcome.test(Html2rss.test(feed_config, **test_args))
|
|
141
141
|
}
|
|
142
142
|
),
|
|
@@ -163,7 +163,7 @@ module Html2rss
|
|
|
163
163
|
# @param items_selector [String, nil]
|
|
164
164
|
# @return [Outcome]
|
|
165
165
|
def scrape_outcome(url:, strategy: 'auto', limit: 25, items_selector: nil)
|
|
166
|
-
wire = Batch.scrape_wire(url:, strategy:
|
|
166
|
+
wire = Batch.scrape_wire(url:, strategy: Option.strategy(strategy), limit:, items_selector:)
|
|
167
167
|
Outcome.scrape(
|
|
168
168
|
items: wire[:items],
|
|
169
169
|
requested_strategy: wire[:strategy],
|
|
@@ -189,7 +189,7 @@ module Html2rss
|
|
|
189
189
|
# @return [Outcome]
|
|
190
190
|
def capture_outcome(url:, strategy: 'auto', items_selector: nil, force: false, topics: nil, title: nil, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
|
|
191
191
|
summary: nil, enhance: nil, limit: nil, max_redirects: nil, max_requests: nil)
|
|
192
|
-
plan =
|
|
192
|
+
plan = Option.strategy(strategy)
|
|
193
193
|
result = Html2rss::Capture.build(
|
|
194
194
|
url,
|
|
195
195
|
strategy: plan,
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
##
|
|
5
|
+
# Edge validation for caller-supplied option values (public API, CLI, MCP).
|
|
6
|
+
# Everything past these checks may trust its types.
|
|
7
|
+
module Option
|
|
8
|
+
class << self
|
|
9
|
+
##
|
|
10
|
+
# @param name [Symbol] option name used in the error message
|
|
11
|
+
# @param value [Object] candidate value
|
|
12
|
+
# @param min [Integer] inclusive lower bound
|
|
13
|
+
# @param max [Integer, nil] inclusive upper bound
|
|
14
|
+
# @param optional [Boolean] whether +nil+ passes through
|
|
15
|
+
# @return [Integer, nil] +value+ when valid
|
|
16
|
+
# @raise [InvalidOption] when +value+ is not an Integer within bounds
|
|
17
|
+
def integer(name, value, min:, max: nil, optional: false)
|
|
18
|
+
return value if value.nil? && optional
|
|
19
|
+
return value if value.is_a?(Integer) && (min..max).cover?(value)
|
|
20
|
+
|
|
21
|
+
raise InvalidOption, "#{name} must be an Integer in #{min}..#{max}, got #{value.inspect}"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
##
|
|
25
|
+
# @param value [Symbol, String, nil] strategy plan name (+nil+ means +:auto+)
|
|
26
|
+
# @return [Symbol] the plan name
|
|
27
|
+
# @raise [InvalidOption] when +value+ is not a registered plan
|
|
28
|
+
def strategy(value)
|
|
29
|
+
name = value.nil? ? FeedPipeline::StrategyPlan::AUTO_NAME : value
|
|
30
|
+
return name.to_sym if FeedPipeline::StrategyPlan.valid?(name)
|
|
31
|
+
|
|
32
|
+
accepted = FeedPipeline::StrategyPlan.accepted_names.join(', ')
|
|
33
|
+
raise InvalidOption, "strategy must be one of #{accepted}, got #{value.inspect}"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
##
|
|
37
|
+
# @param name [Symbol] option name used in the error message
|
|
38
|
+
# @param value [Object] candidate list
|
|
39
|
+
# @return [Array] +value+ when it is a non-empty Array
|
|
40
|
+
# @raise [InvalidOption] otherwise
|
|
41
|
+
def list(name, value)
|
|
42
|
+
return value if value.is_a?(Array) && value.any?
|
|
43
|
+
|
|
44
|
+
raise InvalidOption, "#{name} must be a non-empty Array, got #{value.inspect}"
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -34,14 +34,12 @@ module Html2rss
|
|
|
34
34
|
JS_SHELL_MIN_BODY_BYTES = 8_192
|
|
35
35
|
private_constant :JS_SHELL_MIN_BODY_BYTES
|
|
36
36
|
|
|
37
|
-
module_function
|
|
38
|
-
|
|
39
37
|
##
|
|
40
38
|
# @param url [String]
|
|
41
39
|
# @param strategy [String, Symbol]
|
|
42
40
|
# @param deep [Boolean] when true and strategy is auto, one Botasaurus hop if configured
|
|
43
41
|
# @return [Report]
|
|
44
|
-
def call(url:, strategy: :auto, deep: false)
|
|
42
|
+
def self.call(url:, strategy: :auto, deep: false)
|
|
45
43
|
probe = PageRecon.probe(url, strategy: resolve_inspect_strategy(strategy, deep:))
|
|
46
44
|
recon = probe.result
|
|
47
45
|
response = probe.response
|
|
@@ -56,10 +54,10 @@ module Html2rss
|
|
|
56
54
|
# @param strategy [Symbol, String]
|
|
57
55
|
# @param concurrency [Integer]
|
|
58
56
|
# @return [Array<Report>]
|
|
59
|
-
def batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
|
|
57
|
+
def self.batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
|
|
60
58
|
Batch.map(Array(urls), concurrency:) do |url|
|
|
61
59
|
call(url:, strategy:)
|
|
62
|
-
rescue
|
|
60
|
+
rescue Html2rss::Error => error
|
|
63
61
|
error_report(url, error)
|
|
64
62
|
end
|
|
65
63
|
end
|
|
@@ -67,7 +65,7 @@ module Html2rss
|
|
|
67
65
|
##
|
|
68
66
|
# @param response [Html2rss::RequestService::Response]
|
|
69
67
|
# @return [Hash] redacted XHR capture diagnostics (no query strings)
|
|
70
|
-
def xhr_capture_info(response)
|
|
68
|
+
def self.xhr_capture_info(response)
|
|
71
69
|
captured = response.captured_responses
|
|
72
70
|
{
|
|
73
71
|
count: captured.size,
|
|
@@ -79,7 +77,7 @@ module Html2rss
|
|
|
79
77
|
##
|
|
80
78
|
# @param entry [Hash] captured response hash
|
|
81
79
|
# @return [String, nil] scheme+host+path only
|
|
82
|
-
def redacted_endpoint(entry)
|
|
80
|
+
def self.redacted_endpoint(entry)
|
|
83
81
|
raw = entry['url'] || entry[:url]
|
|
84
82
|
return unless raw
|
|
85
83
|
|
|
@@ -94,7 +92,7 @@ module Html2rss
|
|
|
94
92
|
##
|
|
95
93
|
# @param entry [Hash] captured response hash
|
|
96
94
|
# @return [Boolean]
|
|
97
|
-
def xhr_candidate_articles?(entry)
|
|
95
|
+
def self.xhr_candidate_articles?(entry)
|
|
98
96
|
body = entry['body'] || entry[:body]
|
|
99
97
|
return false unless body.is_a?(String)
|
|
100
98
|
|
|
@@ -107,7 +105,7 @@ module Html2rss
|
|
|
107
105
|
##
|
|
108
106
|
# @param parsed [Object] parsed response body
|
|
109
107
|
# @return [Array<String>, Hash]
|
|
110
|
-
def scraper_info(parsed)
|
|
108
|
+
def self.scraper_info(parsed)
|
|
111
109
|
return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
|
|
112
110
|
|
|
113
111
|
begin
|
data/lib/html2rss/page_recon.rb
CHANGED
|
@@ -158,8 +158,6 @@ module Html2rss
|
|
|
158
158
|
def self.discover_segments(sst, url)
|
|
159
159
|
link_resolver = Scoring::LinkResolver.new(url)
|
|
160
160
|
AutoSource::Segmenter.call(sst, base_url: url, strategy: :list, link_resolver:)
|
|
161
|
-
rescue StandardError
|
|
162
|
-
[]
|
|
163
161
|
end
|
|
164
162
|
|
|
165
163
|
##
|
|
@@ -268,8 +266,6 @@ module Html2rss
|
|
|
268
266
|
|
|
269
267
|
def sst_document
|
|
270
268
|
Html2rss::SST::Normalizer.call(response.body)
|
|
271
|
-
rescue ArgumentError
|
|
272
|
-
nil
|
|
273
269
|
end
|
|
274
270
|
|
|
275
271
|
def segment_stats(sst, page_url)
|
data/lib/html2rss/recon.rb
CHANGED
|
@@ -9,8 +9,6 @@ module Html2rss
|
|
|
9
9
|
# Service that runs reconnaissance on a URL to discover redirect chains,
|
|
10
10
|
# native RSS/Atom feeds, surface categorization, and emit an actionable verdict.
|
|
11
11
|
module Recon # rubocop:disable Metrics/ModuleLength
|
|
12
|
-
module_function
|
|
13
|
-
|
|
14
12
|
Verdict = Data.define(:name)
|
|
15
13
|
|
|
16
14
|
##
|
|
@@ -106,12 +104,12 @@ module Html2rss
|
|
|
106
104
|
# @option options [Integer, nil] :max_redirects optional maximum redirects
|
|
107
105
|
# @option options [Integer, nil] :max_requests optional request budget
|
|
108
106
|
# @return [Html2rss::Recon::Result]
|
|
109
|
-
def call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
107
|
+
def self.call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
110
108
|
url_obj = Url.from_absolute(url)
|
|
111
109
|
|
|
112
110
|
begin
|
|
113
111
|
probe = PageRecon.probe(url_obj, strategy:, **)
|
|
114
|
-
rescue
|
|
112
|
+
rescue Html2rss::Error => error
|
|
115
113
|
return error_result(url_obj, error)
|
|
116
114
|
end
|
|
117
115
|
|
|
@@ -149,7 +147,7 @@ module Html2rss
|
|
|
149
147
|
# @option options [Integer, nil] :max_redirects optional maximum redirects
|
|
150
148
|
# @option options [Integer, nil] :max_requests optional request budget
|
|
151
149
|
# @return [Array<Html2rss::Recon::Result>]
|
|
152
|
-
def batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
|
|
150
|
+
def self.batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
|
|
153
151
|
url_list = urls.map { |u| u.to_s.strip }.reject(&:empty?)
|
|
154
152
|
return [] if url_list.empty?
|
|
155
153
|
|
|
@@ -166,7 +164,7 @@ module Html2rss
|
|
|
166
164
|
# @param session [Html2rss::RequestSession]
|
|
167
165
|
# @param response [Html2rss::RequestService::Response]
|
|
168
166
|
# @return [Array(Html2rss::Url, nil), String, nil)] feed URL and optional discovery error note
|
|
169
|
-
def find_native_feed(url_obj, session, response)
|
|
167
|
+
def self.find_native_feed(url_obj, session, response)
|
|
170
168
|
feed = Syndication::Discovery.best_feed_url(
|
|
171
169
|
page_url: url_obj,
|
|
172
170
|
request_session: session,
|
|
@@ -174,12 +172,12 @@ module Html2rss
|
|
|
174
172
|
html: response.body
|
|
175
173
|
)
|
|
176
174
|
[feed, nil]
|
|
177
|
-
rescue
|
|
175
|
+
rescue Html2rss::Error => error
|
|
178
176
|
[nil, "discovery_error=#{error.class}: #{error.message}"]
|
|
179
177
|
end
|
|
180
178
|
private_class_method :find_native_feed
|
|
181
179
|
|
|
182
|
-
def build_notes(recon, native_feed, response)
|
|
180
|
+
def self.build_notes(recon, native_feed, response)
|
|
183
181
|
notes = []
|
|
184
182
|
notes << "native_rss=#{native_feed}" if native_feed
|
|
185
183
|
if recon.scheme_downgrade
|
|
@@ -192,7 +190,7 @@ module Html2rss
|
|
|
192
190
|
end
|
|
193
191
|
private_class_method :build_notes
|
|
194
192
|
|
|
195
|
-
def determine_verdict(recon, native_feed)
|
|
193
|
+
def self.determine_verdict(recon, native_feed)
|
|
196
194
|
return Verdict[:drop] if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
|
|
197
195
|
return Verdict[:defer] if native_feed
|
|
198
196
|
|
|
@@ -203,7 +201,7 @@ module Html2rss
|
|
|
203
201
|
end
|
|
204
202
|
private_class_method :determine_verdict
|
|
205
203
|
|
|
206
|
-
def error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
|
|
204
|
+
def self.error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
|
|
207
205
|
Result.new(
|
|
208
206
|
requested_url: url_obj,
|
|
209
207
|
final_url: url_obj,
|
|
@@ -226,14 +224,14 @@ module Html2rss
|
|
|
226
224
|
# @param cache_dir [String]
|
|
227
225
|
# @param cache_mutex [Mutex, nil]
|
|
228
226
|
# @return [void]
|
|
229
|
-
def cache_html_body(body, url_obj, cache_dir, cache_mutex)
|
|
227
|
+
def self.cache_html_body(body, url_obj, cache_dir, cache_mutex)
|
|
230
228
|
host = url_obj.host.to_s.delete_prefix('www.')
|
|
231
229
|
host = 'snapshot' if host.empty?
|
|
232
230
|
digest = Digest::SHA256.hexdigest(url_obj.to_s)[0, 12]
|
|
233
231
|
file_path = File.join(cache_dir, "#{host}-#{digest}.html")
|
|
234
232
|
writer = -> { File.write(file_path, body) }
|
|
235
233
|
cache_mutex ? cache_mutex.synchronize(&writer) : writer.call
|
|
236
|
-
rescue
|
|
234
|
+
rescue SystemCallError, IOError => error
|
|
237
235
|
Log.debug("Recon cache failed: #{error.message}")
|
|
238
236
|
end
|
|
239
237
|
private_class_method :cache_html_body
|
|
@@ -79,10 +79,10 @@ module Html2rss
|
|
|
79
79
|
end
|
|
80
80
|
|
|
81
81
|
def self.interstitial_signature_match?(body, signature)
|
|
82
|
-
min_matches = signature
|
|
82
|
+
min_matches = signature[:min_matches]
|
|
83
83
|
matches = 0
|
|
84
84
|
|
|
85
|
-
signature
|
|
85
|
+
signature[:patterns].each do |pattern|
|
|
86
86
|
matches += 1 if pattern.match?(body)
|
|
87
87
|
return true if matches >= min_matches
|
|
88
88
|
end
|
|
@@ -91,11 +91,7 @@ module Html2rss
|
|
|
91
91
|
end
|
|
92
92
|
private_class_method :interstitial_signature_match?
|
|
93
93
|
|
|
94
|
-
def self.normalize_body(body)
|
|
95
|
-
body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
|
|
96
|
-
rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
|
|
97
|
-
body.to_s.force_encoding(Encoding::UTF_8).scrub
|
|
98
|
-
end
|
|
94
|
+
def self.normalize_body(body) = body.to_s.scrub
|
|
99
95
|
private_class_method :normalize_body
|
|
100
96
|
end
|
|
101
97
|
end
|
|
@@ -65,7 +65,7 @@ module Html2rss
|
|
|
65
65
|
return ctx.url if final_url.nil?
|
|
66
66
|
|
|
67
67
|
Html2rss::Url.from_absolute(final_url)
|
|
68
|
-
rescue
|
|
68
|
+
rescue Html2rss::Url::InvalidUrlError
|
|
69
69
|
ctx.url
|
|
70
70
|
end
|
|
71
71
|
|
|
@@ -117,7 +117,7 @@ module Html2rss
|
|
|
117
117
|
raise BotasaurusConfigurationError, 'BOTASAURUS_SCRAPER_URL is required for strategy=botasaurus.'
|
|
118
118
|
end
|
|
119
119
|
Html2rss::Url.for_channel(configured)
|
|
120
|
-
rescue
|
|
120
|
+
rescue Html2rss::Url::InvalidUrlError => error
|
|
121
121
|
raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
|
|
122
122
|
end
|
|
123
123
|
end
|
|
@@ -11,7 +11,7 @@ module Html2rss
|
|
|
11
11
|
# @param max_requests [Integer] maximum HTTP request slots
|
|
12
12
|
# @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
|
|
13
13
|
def initialize(max_requests:, total_timeout_seconds: nil)
|
|
14
|
-
|
|
14
|
+
Option.integer(:max_requests, max_requests, min: 1)
|
|
15
15
|
|
|
16
16
|
@remaining_requests = max_requests
|
|
17
17
|
@start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
@@ -25,33 +25,20 @@ module Html2rss
|
|
|
25
25
|
# @return [Integer] remaining request slots after consumption
|
|
26
26
|
# @raise [RequestBudgetExceeded] if no request slots remain
|
|
27
27
|
def consume!
|
|
28
|
-
consume_request!
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
##
|
|
32
|
-
# Consumes one HTTP request slot.
|
|
33
|
-
#
|
|
34
|
-
# @return [Integer] remaining request slots after consumption
|
|
35
|
-
# @raise [RequestBudgetExceeded] if no request slots remain
|
|
36
|
-
def consume_request!
|
|
37
28
|
@mutex.synchronize do
|
|
38
29
|
raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining_requests.zero?
|
|
39
30
|
|
|
40
31
|
@remaining_requests -= 1
|
|
41
32
|
end
|
|
42
33
|
end
|
|
43
|
-
|
|
44
|
-
##
|
|
45
|
-
# @return [Integer] HTTP request slots still available
|
|
46
|
-
def remaining
|
|
47
|
-
remaining_requests
|
|
48
|
-
end
|
|
34
|
+
alias consume_request! consume!
|
|
49
35
|
|
|
50
36
|
##
|
|
51
37
|
# @return [Integer] HTTP request slots still available
|
|
52
38
|
def remaining_requests
|
|
53
39
|
@mutex.synchronize { @remaining_requests }
|
|
54
40
|
end
|
|
41
|
+
alias remaining remaining_requests
|
|
55
42
|
|
|
56
43
|
##
|
|
57
44
|
# @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
|
|
@@ -91,14 +78,6 @@ module Html2rss
|
|
|
91
78
|
def effective_timeout_ms(fallback:)
|
|
92
79
|
(effective_timeout_seconds(fallback:) * 1000).to_i
|
|
93
80
|
end
|
|
94
|
-
|
|
95
|
-
private
|
|
96
|
-
|
|
97
|
-
def validate_slot_limits!(max_requests:)
|
|
98
|
-
return if max_requests.is_a?(Integer) && max_requests.positive?
|
|
99
|
-
|
|
100
|
-
raise ArgumentError, 'max_requests must be positive'
|
|
101
|
-
end
|
|
102
81
|
end
|
|
103
82
|
end
|
|
104
83
|
end
|
|
@@ -7,56 +7,58 @@ require 'zlib'
|
|
|
7
7
|
module Html2rss
|
|
8
8
|
class RequestService
|
|
9
9
|
# Decodes gzip/brotli bodies when Content-Encoding is missing or the type is octet-stream.
|
|
10
|
-
|
|
10
|
+
class CompressedBody
|
|
11
11
|
# Gzip stream magic (RFC 1952).
|
|
12
12
|
GZIP_MAGIC = "\x1F\x8B".b.freeze
|
|
13
13
|
# HTML prefix for unlabeled brotli trial decode. Same object as {Response::HTML_BODY_SNIFF}.
|
|
14
14
|
HTML_BODY_SNIFF = Response::HTML_BODY_SNIFF
|
|
15
15
|
|
|
16
|
-
module_function
|
|
17
|
-
|
|
18
16
|
# @param body [String, nil] raw response body
|
|
19
17
|
# @param headers [Hash] response headers (Content-Encoding / Content-Type)
|
|
20
18
|
# @return [String, nil] decoded body when compressed; otherwise the original body
|
|
21
|
-
def decode(body, headers: {})
|
|
22
|
-
return body if body.nil? || body.empty?
|
|
23
|
-
|
|
19
|
+
def self.decode(body, headers: {})
|
|
24
20
|
raw = body.to_s.b
|
|
25
|
-
|
|
21
|
+
new(raw, headers).decoded_body || body
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# @param raw [String] binary response body
|
|
25
|
+
# @param headers [Hash] response headers (Content-Encoding / Content-Type)
|
|
26
|
+
def initialize(raw, headers)
|
|
27
|
+
@raw = raw
|
|
28
|
+
@headers = headers
|
|
26
29
|
end
|
|
30
|
+
private_class_method :new
|
|
27
31
|
|
|
28
|
-
|
|
29
|
-
|
|
32
|
+
# @return [String, nil] inflated body, or nil when the bytes are not a recognized stream
|
|
33
|
+
def decoded_body
|
|
34
|
+
decode_labeled || decode_unlabeled
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def decode_labeled
|
|
40
|
+
encodings = content_encodings
|
|
30
41
|
return if encodings.empty?
|
|
31
42
|
|
|
32
|
-
encodings.reverse.reduce(raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
|
|
43
|
+
encodings.reverse.reduce(@raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
|
|
33
44
|
end
|
|
34
|
-
module_function :decode_labeled
|
|
35
|
-
private_class_method :decode_labeled
|
|
36
45
|
|
|
37
|
-
def decode_unlabeled
|
|
38
|
-
return uncompress_gzip(raw) if raw.start_with?(GZIP_MAGIC)
|
|
39
|
-
return unless octet_stream?
|
|
46
|
+
def decode_unlabeled
|
|
47
|
+
return uncompress_gzip(@raw) if @raw.start_with?(GZIP_MAGIC)
|
|
48
|
+
return unless octet_stream?
|
|
40
49
|
|
|
41
|
-
try_brotli(raw)
|
|
50
|
+
try_brotli(@raw)
|
|
42
51
|
end
|
|
43
|
-
module_function :decode_unlabeled
|
|
44
|
-
private_class_method :decode_unlabeled
|
|
45
52
|
|
|
46
53
|
def inflate(raw, encoding)
|
|
47
54
|
case encoding
|
|
48
|
-
when 'gzip'
|
|
49
|
-
return raw unless raw.start_with?(GZIP_MAGIC)
|
|
50
|
-
|
|
51
|
-
uncompress_gzip(raw)
|
|
55
|
+
when 'gzip' then uncompress_gzip(raw)
|
|
52
56
|
when 'br' then try_brotli(raw, require_html: false)
|
|
53
57
|
when 'deflate' then inflate_deflate(raw)
|
|
54
58
|
end
|
|
55
59
|
rescue Zlib::Error, Brotli::Error, ArgumentError
|
|
56
60
|
nil
|
|
57
61
|
end
|
|
58
|
-
module_function :inflate
|
|
59
|
-
private_class_method :inflate
|
|
60
62
|
|
|
61
63
|
def inflate_deflate(raw)
|
|
62
64
|
inflater = nil
|
|
@@ -67,51 +69,37 @@ module Html2rss
|
|
|
67
69
|
ensure
|
|
68
70
|
inflater&.close
|
|
69
71
|
end
|
|
70
|
-
module_function :inflate_deflate
|
|
71
|
-
private_class_method :inflate_deflate
|
|
72
72
|
|
|
73
73
|
def uncompress_gzip(raw)
|
|
74
|
-
Zlib::GzipReader.wrap(StringIO.new(raw),
|
|
74
|
+
Zlib::GzipReader.wrap(StringIO.new(raw), &:read).b
|
|
75
75
|
rescue Zlib::Error
|
|
76
76
|
nil
|
|
77
77
|
end
|
|
78
|
-
module_function :uncompress_gzip
|
|
79
|
-
private_class_method :uncompress_gzip
|
|
80
78
|
|
|
81
79
|
def try_brotli(raw, require_html: true)
|
|
82
80
|
inflated = Brotli.inflate(raw)
|
|
83
81
|
return inflated unless require_html
|
|
84
|
-
return inflated if inflated.b.match?(HTML_BODY_SNIFF)
|
|
85
82
|
|
|
86
|
-
nil
|
|
83
|
+
inflated.b.match?(HTML_BODY_SNIFF) ? inflated : nil
|
|
87
84
|
rescue Brotli::Error, ArgumentError
|
|
88
85
|
nil
|
|
89
86
|
end
|
|
90
|
-
module_function :try_brotli
|
|
91
|
-
private_class_method :try_brotli
|
|
92
87
|
|
|
93
|
-
def content_encodings
|
|
94
|
-
header(
|
|
88
|
+
def content_encodings
|
|
89
|
+
header('content-encoding')
|
|
95
90
|
.to_s
|
|
96
91
|
.split(',')
|
|
97
|
-
.
|
|
92
|
+
.map { |val| val.strip.downcase }
|
|
93
|
+
.reject(&:empty?)
|
|
98
94
|
end
|
|
99
|
-
module_function :content_encodings
|
|
100
|
-
private_class_method :content_encodings
|
|
101
95
|
|
|
102
|
-
def octet_stream?
|
|
103
|
-
header(
|
|
96
|
+
def octet_stream?
|
|
97
|
+
header('content-type').to_s.downcase.include?('application/octet-stream')
|
|
104
98
|
end
|
|
105
|
-
module_function :octet_stream?
|
|
106
|
-
private_class_method :octet_stream?
|
|
107
99
|
|
|
108
|
-
def header(
|
|
109
|
-
headers.
|
|
110
|
-
headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
|
|
111
|
-
end
|
|
100
|
+
def header(name)
|
|
101
|
+
@headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
|
|
112
102
|
end
|
|
113
|
-
module_function :header
|
|
114
|
-
private_class_method :header
|
|
115
103
|
end
|
|
116
104
|
end
|
|
117
105
|
end
|
|
@@ -87,9 +87,7 @@ module Html2rss
|
|
|
87
87
|
end
|
|
88
88
|
|
|
89
89
|
def normalize_request(request)
|
|
90
|
-
|
|
91
|
-
HashUtil.assert_symbol_keys!(normalized, context: 'request')
|
|
92
|
-
normalized
|
|
90
|
+
HashUtil.deep_symbolize_keys(request, context: 'request')
|
|
93
91
|
end
|
|
94
92
|
end
|
|
95
93
|
end
|
|
@@ -15,6 +15,9 @@ module Html2rss
|
|
|
15
15
|
|
|
16
16
|
def skip_postflight? = true
|
|
17
17
|
|
|
18
|
+
# Local files never open a socket; keep filesystem errors as wire classes.
|
|
19
|
+
def connection_error?(_error) = false
|
|
20
|
+
|
|
18
21
|
##
|
|
19
22
|
# @return [Response] the mock response wrapped around the file contents
|
|
20
23
|
# @raise [ArgumentError] if the local file path is missing
|
|
@@ -22,7 +25,6 @@ module Html2rss
|
|
|
22
25
|
def fetch
|
|
23
26
|
file_path = ctx.request[:local_file_path]
|
|
24
27
|
raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
|
|
25
|
-
raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)
|
|
26
28
|
|
|
27
29
|
body = File.read(file_path)
|
|
28
30
|
Response.new(
|
|
@@ -68,7 +68,7 @@ module Html2rss
|
|
|
68
68
|
def validate_remote_ip!(ip:, url:)
|
|
69
69
|
return if allow_private_networks?
|
|
70
70
|
|
|
71
|
-
parsed_ip = ip
|
|
71
|
+
parsed_ip = parse_ip(ip.to_s)
|
|
72
72
|
raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
|
|
73
73
|
return unless blocked_ip?(parsed_ip)
|
|
74
74
|
|
|
@@ -122,7 +122,8 @@ module Html2rss
|
|
|
122
122
|
end
|
|
123
123
|
|
|
124
124
|
def blocked_ip?(address)
|
|
125
|
-
|
|
125
|
+
native = address.native
|
|
126
|
+
BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
|
|
126
127
|
end
|
|
127
128
|
end
|
|
128
129
|
end
|