ficon 0.5 → 0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/ficon/version.rb +1 -1
- data/lib/ficon.rb +60 -30
- data/test/ficon_test.rb +112 -56
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 84e88d986bdad619358041cc478cf34418278ebdf74cb6ef12df7b653734d9dd
|
|
4
|
+
data.tar.gz: 1ed0fbcd8c01895246099fea8e26f35e358e9f4a46ecc5996add3ed6f8d193d6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 00d5013559e4a203ced2c780582a6f705a3883966d59c4f28c4a5bd69aac9a9a425cd3420adca17dec86993cca81825e80ba40ca1c4d633c40423c6c659bf4df
|
|
7
|
+
data.tar.gz: bb7311ea20869d3394789da06cfb759a64713dfc82eadb419e55561b98c49fac3797b741d8d4ab529fbae63cfbc6ed58ae22fa759404410d027f2a506a8538a2
|
data/lib/ficon/version.rb
CHANGED
data/lib/ficon.rb
CHANGED
|
@@ -1,18 +1,17 @@
|
|
|
1
|
-
require
|
|
2
|
-
require
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require
|
|
6
|
-
require "debug"
|
|
1
|
+
require 'net/http'
|
|
2
|
+
require 'nokogiri'
|
|
3
|
+
require 'uri'
|
|
4
|
+
require 'addressable/uri'
|
|
5
|
+
require 'resolv'
|
|
7
6
|
|
|
8
|
-
require_relative
|
|
9
|
-
require_relative
|
|
10
|
-
require_relative
|
|
7
|
+
require_relative 'ficon/version'
|
|
8
|
+
require_relative 'ficon/image'
|
|
9
|
+
require_relative 'ficon/cache'
|
|
11
10
|
|
|
12
11
|
class Ficon
|
|
13
12
|
attr_reader :site, :final_uri, :url_status
|
|
14
13
|
attr_accessor :user_agent
|
|
15
|
-
|
|
14
|
+
|
|
16
15
|
# URL health status constants
|
|
17
16
|
ALIVE = 'alive'
|
|
18
17
|
DEAD = 'dead'
|
|
@@ -39,10 +38,10 @@ class Ficon
|
|
|
39
38
|
@data ||= cache.data
|
|
40
39
|
|
|
41
40
|
if @data.nil? && response
|
|
42
|
-
@data = response.body.force_encoding(
|
|
41
|
+
@data = response.body.force_encoding('UTF-8')
|
|
43
42
|
cache.data = @data
|
|
44
|
-
cache.etag = response[
|
|
45
|
-
cache.not_before = response[
|
|
43
|
+
cache.etag = response['etag'] if response['etag']
|
|
44
|
+
cache.not_before = response['last-modified'] if response['last-modified']
|
|
46
45
|
end
|
|
47
46
|
|
|
48
47
|
@doc ||= Nokogiri::HTML(@data)
|
|
@@ -55,7 +54,7 @@ class Ficon
|
|
|
55
54
|
puts "#{e.inspect}"
|
|
56
55
|
puts "#{e.backtrace.join('\n')}"
|
|
57
56
|
else
|
|
58
|
-
puts
|
|
57
|
+
puts 'Please prepend http:// or https:// to the URL'
|
|
59
58
|
end
|
|
60
59
|
nil
|
|
61
60
|
rescue RuntimeError => e
|
|
@@ -101,7 +100,8 @@ class Ficon
|
|
|
101
100
|
end
|
|
102
101
|
|
|
103
102
|
def other_page_data(document)
|
|
104
|
-
@site[:title] =
|
|
103
|
+
@site[:title] =
|
|
104
|
+
document.at_xpath("//meta[@property='og:title']/@content")&.value || document.at_xpath('//title')&.text&.strip
|
|
105
105
|
@site[:description] = document.at_xpath("//meta[@property='og:description']/@content")&.value
|
|
106
106
|
canonical = document.at_xpath("//link[@rel='canonical']/@href")&.value
|
|
107
107
|
@site[:canonical] = canonical unless canonical == @uri.to_s
|
|
@@ -114,27 +114,36 @@ class Ficon
|
|
|
114
114
|
tile_color = doc.at_xpath("//meta[@name='msapplication-TileColor']/@content")&.value
|
|
115
115
|
|
|
116
116
|
paths = "//meta[@name='msapplication-TileImage']|//link[@type='image/ico' or @type='image/vnd.microsoft.icon']|//link[@rel='icon' or @rel='shortcut icon' or @rel='apple-touch-icon-precomposed' or @rel='apple-touch-icon']"
|
|
117
|
-
results += doc.xpath(paths).collect
|
|
117
|
+
results += doc.xpath(paths).collect do |e|
|
|
118
|
+
e.values.select do |v|
|
|
119
|
+
v =~ /\.png$|\.jpg$|\.gif$|\.ico$|\.svg$|\.ico\?\d*$/
|
|
120
|
+
end
|
|
121
|
+
end.flatten.collect { |v| v[/^http/] || v[%r{^/}] ? v : '/' + v }
|
|
118
122
|
|
|
119
123
|
results.collect { |result| normalise(uri, result) }.uniq.collect do |url|
|
|
120
124
|
# Check if this is a tile image to pass the color
|
|
121
|
-
is_tile = doc.at_xpath("//meta[@name='msapplication-TileImage' and @content='#{url}' or @content='#{url.sub(
|
|
125
|
+
is_tile = doc.at_xpath("//meta[@name='msapplication-TileImage' and @content='#{url}' or @content='#{url.sub(
|
|
126
|
+
uri.to_s, ''
|
|
127
|
+
)}']")
|
|
122
128
|
Image.new(url, is_tile ? tile_color : nil)
|
|
123
129
|
end.sort_by(&:area).reverse
|
|
124
130
|
end
|
|
125
131
|
|
|
126
132
|
def self.page_images(uri, doc)
|
|
127
133
|
doc.xpath("//meta[@property='og:image']")
|
|
128
|
-
|
|
129
|
-
|
|
134
|
+
.collect { |e| e.values.reject(&:empty?) }.flatten
|
|
135
|
+
.collect { |v| v[/^http/] || v[%r{^/}] ? v : '/' + v }.collect do |result|
|
|
136
|
+
normalise(uri,
|
|
137
|
+
result)
|
|
138
|
+
end.uniq.collect { |i| Image.new(i) }.sort_by(&:area).reverse
|
|
130
139
|
end
|
|
131
140
|
|
|
132
141
|
def self.normalise(base, candidate)
|
|
133
142
|
parsed_candidate = URI(candidate)
|
|
134
143
|
base = URI(base) unless base.is_a? URI
|
|
135
144
|
|
|
136
|
-
parsed_candidate.host = base.host if parsed_candidate.host.nil?
|
|
137
|
-
parsed_candidate.scheme = base.scheme if parsed_candidate.scheme.nil?
|
|
145
|
+
parsed_candidate.host = base.host if parsed_candidate.host.nil? # Set relative URLs to absolute
|
|
146
|
+
parsed_candidate.scheme = base.scheme if parsed_candidate.scheme.nil? # Set the schema if missing
|
|
138
147
|
|
|
139
148
|
parsed_candidate.to_s
|
|
140
149
|
end
|
|
@@ -171,25 +180,25 @@ class Ficon
|
|
|
171
180
|
|
|
172
181
|
def fetch_url(uri, redirect_limit = 5)
|
|
173
182
|
uri = URI(uri) unless uri.is_a?(URI)
|
|
174
|
-
|
|
183
|
+
|
|
175
184
|
if redirect_limit <= 0
|
|
176
185
|
@url_status = DEAD
|
|
177
|
-
raise
|
|
186
|
+
raise 'Too many redirects'
|
|
178
187
|
end
|
|
179
188
|
|
|
180
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme ==
|
|
189
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
|
|
181
190
|
http.read_timeout = 10
|
|
182
191
|
http.open_timeout = 5
|
|
183
192
|
request = Net::HTTP::Get.new(uri)
|
|
184
|
-
request[
|
|
193
|
+
request['User-Agent'] = @user_agent
|
|
185
194
|
response = http.request(request)
|
|
186
|
-
|
|
195
|
+
|
|
187
196
|
# Set status based on response
|
|
188
197
|
@url_status = classify_response_status(response)
|
|
189
|
-
|
|
198
|
+
|
|
190
199
|
case response
|
|
191
200
|
when Net::HTTPRedirection
|
|
192
|
-
location = response[
|
|
201
|
+
location = response['location']
|
|
193
202
|
if location
|
|
194
203
|
new_uri = URI.join(uri.to_s, location)
|
|
195
204
|
@final_uri = Addressable::URI.parse(new_uri.to_s)
|
|
@@ -198,11 +207,32 @@ class Ficon
|
|
|
198
207
|
else
|
|
199
208
|
@final_uri = Addressable::URI.parse(uri.to_s)
|
|
200
209
|
end
|
|
201
|
-
|
|
210
|
+
|
|
202
211
|
response
|
|
203
212
|
end
|
|
204
|
-
rescue => e
|
|
213
|
+
rescue StandardError => e
|
|
205
214
|
@url_status = classify_exception_status(e)
|
|
215
|
+
|
|
216
|
+
# If HTTP request failed and we're using HTTP, try HTTPS automatically
|
|
217
|
+
if uri.scheme == 'http' &&
|
|
218
|
+
!uri.to_s.include?('://localhost') &&
|
|
219
|
+
!uri.host.match?(/^\d+\.\d+\.\d+\.\d+$/)
|
|
220
|
+
puts "HTTP request failed, trying HTTPS for #{uri}"
|
|
221
|
+
https_uri = uri.dup
|
|
222
|
+
https_uri.scheme = 'https'
|
|
223
|
+
https_uri.port = 443 if https_uri.port == 80
|
|
224
|
+
|
|
225
|
+
begin
|
|
226
|
+
https_response = fetch_url(https_uri, redirect_limit)
|
|
227
|
+
if https_response
|
|
228
|
+
puts 'HTTPS request succeeded, using HTTPS URL'
|
|
229
|
+
return https_response
|
|
230
|
+
end
|
|
231
|
+
rescue StandardError => https_error
|
|
232
|
+
puts "HTTPS fallback also failed: #{https_error.inspect}"
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
206
236
|
puts "Failed to fetch #{uri}: #{e.inspect}"
|
|
207
237
|
nil
|
|
208
238
|
end
|
data/test/ficon_test.rb
CHANGED
|
@@ -1,93 +1,92 @@
|
|
|
1
|
-
#require 'rubygems'
|
|
2
|
-
require
|
|
3
|
-
require
|
|
1
|
+
# require 'rubygems'
|
|
2
|
+
require "debug"
|
|
3
|
+
require "resolv"
|
|
4
4
|
|
|
5
5
|
require "minitest/autorun"
|
|
6
6
|
|
|
7
7
|
PathHere = File.dirname(__FILE__)
|
|
8
8
|
$LOAD_PATH.unshift File.join(PathHere, "..", "lib")
|
|
9
9
|
|
|
10
|
-
require
|
|
11
|
-
|
|
12
|
-
SiteTests = []
|
|
13
|
-
SiteTests << { html: %Q{<meta name="msapplication-TileImage" content="https://s.yimg.com/pw/images/favicon-msapplication-tileimage.png"/> }, value: 'https://s.yimg.com/pw/images/favicon-msapplication-tileimage.png' }
|
|
14
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" type="image/ico" href="https://s.yimg.com/pw/favicon.ico"> }, value: 'https://s.yimg.com/pw/favicon.ico' }
|
|
15
|
-
SiteTests << { html: %Q{<link href="/apple-touch-icon.png" rel="apple-touch-icon-precomposed">}, value: 'https://site.com/apple-touch-icon.png' }
|
|
16
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" href="/wp-content/themes/torrentfreakredux/assets/img/icons/favicon.png">}, value: 'https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/favicon.png' }
|
|
17
|
-
SiteTests << { html: %Q{<link rel="apple-touch-icon-precomposed" href="/wp-content/themes/torrentfreakredux/assets/img/icons/57.png">}, value: 'https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/57.png' }
|
|
18
|
-
SiteTests << { html: %Q{<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/wp-content/themes/torrentfreakredux/assets/img/icons/114.png">}, value: 'https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/114.png' }
|
|
19
|
-
SiteTests << { html: %Q{<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/wp-content/themes/torrentfreakredux/assets/img/icons/72.png">}, value: 'https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/72.png' }
|
|
20
|
-
SiteTests << { html: %Q{<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/wp-content/themes/torrentfreakredux/assets/img/icons/144.png">}, value: 'https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/144.png' }
|
|
21
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" href="/favicon.png">}, value: 'https://site.com/favicon.png' }
|
|
22
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" href="favicon.ico" />}, value: 'https://site.com/favicon.ico' }
|
|
23
|
-
SiteTests << { html: %Q{<link rel="apple-touch-icon" href="/apple-touch-icon.png">}, value: 'https://site.com/apple-touch-icon.png' }
|
|
24
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" href="http://example.com/myicon.ico" />}, value: 'http://example.com/myicon.ico'}
|
|
25
|
-
SiteTests << { html: %Q{<link rel="icon" href="http://example.com/image.ico" />}, value: 'http://example.com/image.ico' }
|
|
26
|
-
SiteTests << { html: %Q{<link rel="icon" type="image/vnd.microsoft.icon" href="http://example.com/image.ico" />}, value: 'http://example.com/image.ico' }
|
|
27
|
-
SiteTests << { html: %Q{<link rel="icon" type="image/png" href="http://example.com/image.png" />}, value: 'http://example.com/image.png' }
|
|
28
|
-
SiteTests << { html: %Q{<link rel="icon" type="image/gif" href="http://example.com/image.gif" />}, value: 'http://example.com/image.gif' }
|
|
29
|
-
SiteTests << { html: %Q{<link rel="icon" type="image/x-icon" href="http://example.com/image.ico"/>}, value: 'http://example.com/image.ico' }
|
|
30
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" href="https://fbstatic-a.akamaihd.net/rsrc.php/yl/r/H3nktOa7ZMg.ico" />}, value: 'https://fbstatic-a.akamaihd.net/rsrc.php/yl/r/H3nktOa7ZMg.ico' }
|
|
31
|
-
SiteTests << { html: %Q{<link rel="icon" type="image/vnd.microsoft.icon" href="/viconline/img/favicon.ico?1393375504" />}, value: 'https://site.com/viconline/img/favicon.ico?1393375504' }
|
|
32
|
-
SiteTests << { html: %Q{<link rel="shortcut icon" type="image/x-icon" href="/viconline/img/favicon.ico?1393375504" />}, value: 'https://site.com/viconline/img/favicon.ico?1393375504' }
|
|
33
|
-
SiteTests << { html: %Q{<meta name="msapplication-TileImage" content="/win8-tile-144.png"/><meta name="msapplication-TileColor" content="#00aced"/>}, value: 'https://site.com/win8-tile-144.png' }
|
|
34
|
-
|
|
35
|
-
PageTests = []
|
|
36
|
-
PageTests << { html: %Q{<meta property="og:image" content="https://www.facebook.com/images/fb_icon_325x325.png" />}, value: 'https://www.facebook.com/images/fb_icon_325x325.png' }
|
|
10
|
+
require "ficon"
|
|
37
11
|
|
|
12
|
+
SiteTests = []
|
|
13
|
+
SiteTests << {html: %(<meta name="msapplication-TileImage" content="https://s.yimg.com/pw/images/favicon-msapplication-tileimage.png"/> ), value: "https://s.yimg.com/pw/images/favicon-msapplication-tileimage.png"}
|
|
14
|
+
SiteTests << {html: %(<link rel="shortcut icon" type="image/ico" href="https://s.yimg.com/pw/favicon.ico"> ), value: "https://s.yimg.com/pw/favicon.ico"}
|
|
15
|
+
SiteTests << {html: %(<link href="/apple-touch-icon.png" rel="apple-touch-icon-precomposed">), value: "https://site.com/apple-touch-icon.png"}
|
|
16
|
+
SiteTests << {html: %(<link rel="shortcut icon" href="/wp-content/themes/torrentfreakredux/assets/img/icons/favicon.png">), value: "https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/favicon.png"}
|
|
17
|
+
SiteTests << {html: %(<link rel="apple-touch-icon-precomposed" href="/wp-content/themes/torrentfreakredux/assets/img/icons/57.png">), value: "https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/57.png"}
|
|
18
|
+
SiteTests << {html: %(<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/wp-content/themes/torrentfreakredux/assets/img/icons/114.png">), value: "https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/114.png"}
|
|
19
|
+
SiteTests << {html: %(<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/wp-content/themes/torrentfreakredux/assets/img/icons/72.png">), value: "https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/72.png"}
|
|
20
|
+
SiteTests << {html: %(<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/wp-content/themes/torrentfreakredux/assets/img/icons/144.png">), value: "https://site.com/wp-content/themes/torrentfreakredux/assets/img/icons/144.png"}
|
|
21
|
+
SiteTests << {html: %(<link rel="shortcut icon" href="/favicon.png">), value: "https://site.com/favicon.png"}
|
|
22
|
+
SiteTests << {html: %(<link rel="shortcut icon" href="favicon.ico" />), value: "https://site.com/favicon.ico"}
|
|
23
|
+
SiteTests << {html: %(<link rel="apple-touch-icon" href="/apple-touch-icon.png">), value: "https://site.com/apple-touch-icon.png"}
|
|
24
|
+
SiteTests << {html: %(<link rel="shortcut icon" href="http://example.com/myicon.ico" />), value: "http://example.com/myicon.ico"}
|
|
25
|
+
SiteTests << {html: %(<link rel="icon" href="http://example.com/image.ico" />), value: "http://example.com/image.ico"}
|
|
26
|
+
SiteTests << {html: %(<link rel="icon" type="image/vnd.microsoft.icon" href="http://example.com/image.ico" />), value: "http://example.com/image.ico"}
|
|
27
|
+
SiteTests << {html: %(<link rel="icon" type="image/png" href="http://example.com/image.png" />), value: "http://example.com/image.png"}
|
|
28
|
+
SiteTests << {html: %(<link rel="icon" type="image/gif" href="http://example.com/image.gif" />), value: "http://example.com/image.gif"}
|
|
29
|
+
SiteTests << {html: %(<link rel="icon" type="image/x-icon" href="http://example.com/image.ico"/>), value: "http://example.com/image.ico"}
|
|
30
|
+
SiteTests << {html: %(<link rel="shortcut icon" href="https://fbstatic-a.akamaihd.net/rsrc.php/yl/r/H3nktOa7ZMg.ico" />), value: "https://fbstatic-a.akamaihd.net/rsrc.php/yl/r/H3nktOa7ZMg.ico"}
|
|
31
|
+
SiteTests << {html: %(<link rel="icon" type="image/vnd.microsoft.icon" href="/viconline/img/favicon.ico?1393375504" />), value: "https://site.com/viconline/img/favicon.ico?1393375504"}
|
|
32
|
+
SiteTests << {html: %(<link rel="shortcut icon" type="image/x-icon" href="/viconline/img/favicon.ico?1393375504" />), value: "https://site.com/viconline/img/favicon.ico?1393375504"}
|
|
33
|
+
SiteTests << {html: %(<meta name="msapplication-TileImage" content="/win8-tile-144.png"/><meta name="msapplication-TileColor" content="#00aced"/>), value: "https://site.com/win8-tile-144.png"}
|
|
34
|
+
|
|
35
|
+
PageTests = []
|
|
36
|
+
PageTests << {html: %(<meta property="og:image" content="https://www.facebook.com/images/fb_icon_325x325.png" />), value: "https://www.facebook.com/images/fb_icon_325x325.png"}
|
|
38
37
|
|
|
39
38
|
class FiconTest < Minitest::Test
|
|
40
|
-
ENV[
|
|
39
|
+
ENV["FICON_DB"] = File.join(File.dirname(__FILE__), "test.db")
|
|
41
40
|
def test_html_chunks
|
|
42
41
|
SiteTests.each do |t|
|
|
43
|
-
result = Ficon.site_images(
|
|
44
|
-
assert result&.url == t[:value], "Seaching |#{t[:html]}| expected #{t[:value]}, got #{result}"
|
|
42
|
+
result = Ficon.site_images("https://site.com", Nokogiri::HTML(t[:html]))[0]
|
|
43
|
+
assert result&.url == t[:value], "Seaching |#{t[:html]}| expected #{t[:value]}, got #{result}"
|
|
45
44
|
end
|
|
46
45
|
PageTests.each do |t|
|
|
47
|
-
result = Ficon.page_images(
|
|
48
|
-
assert result&.url == t[:value], "Seaching |#{t[:html]}| expected #{t[:value]}, got #{result}"
|
|
46
|
+
result = Ficon.page_images("https://site.com", Nokogiri::HTML(t[:html]))[0]
|
|
47
|
+
assert result&.url == t[:value], "Seaching |#{t[:html]}| expected #{t[:value]}, got #{result}"
|
|
49
48
|
end
|
|
50
49
|
end
|
|
51
50
|
|
|
52
51
|
def test_tile_color_extraction
|
|
53
|
-
html = %
|
|
54
|
-
result = Ficon.site_images(
|
|
55
|
-
assert_equal
|
|
56
|
-
assert_equal
|
|
52
|
+
html = %(<meta name="msapplication-TileImage" content="/win8-tile-144.png"/><meta name="msapplication-TileColor" content="#00aced"/>)
|
|
53
|
+
result = Ficon.site_images("https://site.com", Nokogiri::HTML(html))[0]
|
|
54
|
+
assert_equal "https://site.com/win8-tile-144.png", result.url
|
|
55
|
+
assert_equal "#00aced", result.tile_color
|
|
57
56
|
end
|
|
58
57
|
|
|
59
58
|
def test_custom_user_agent
|
|
60
59
|
# Test default user agent
|
|
61
|
-
ficon_default = Ficon.new(
|
|
60
|
+
ficon_default = Ficon.new("https://example.com")
|
|
62
61
|
assert_match(/^FiconBot\/0\.\d+/, ficon_default.user_agent)
|
|
63
|
-
|
|
62
|
+
|
|
64
63
|
# Test custom user agent
|
|
65
|
-
custom_agent =
|
|
66
|
-
ficon_custom = Ficon.new(
|
|
64
|
+
custom_agent = "MyApp/1.0 (Custom Bot)"
|
|
65
|
+
ficon_custom = Ficon.new("https://example.com", user_agent: custom_agent)
|
|
67
66
|
assert_equal custom_agent, ficon_custom.user_agent
|
|
68
|
-
|
|
67
|
+
|
|
69
68
|
# Test user agent can be changed after initialization
|
|
70
|
-
ficon_custom.user_agent =
|
|
71
|
-
assert_equal
|
|
69
|
+
ficon_custom.user_agent = "Changed/2.0"
|
|
70
|
+
assert_equal "Changed/2.0", ficon_custom.user_agent
|
|
72
71
|
end
|
|
73
72
|
|
|
74
73
|
def test_response_status_classification
|
|
75
|
-
ficon = Ficon.new(
|
|
76
|
-
|
|
74
|
+
ficon = Ficon.new("https://example.com")
|
|
75
|
+
|
|
77
76
|
# Test ALIVE status (2xx)
|
|
78
77
|
assert_equal Ficon::ALIVE, ficon.classify_response_status(mock_response(200))
|
|
79
78
|
assert_equal Ficon::ALIVE, ficon.classify_response_status(mock_response(201))
|
|
80
79
|
assert_equal Ficon::ALIVE, ficon.classify_response_status(mock_response(299))
|
|
81
|
-
|
|
80
|
+
|
|
82
81
|
# Test DEAD status (404, 410)
|
|
83
82
|
assert_equal Ficon::DEAD, ficon.classify_response_status(mock_response(404))
|
|
84
83
|
assert_equal Ficon::DEAD, ficon.classify_response_status(mock_response(410))
|
|
85
|
-
|
|
84
|
+
|
|
86
85
|
# Test BLOCKED status (401, 403, 429)
|
|
87
86
|
assert_equal Ficon::BLOCKED, ficon.classify_response_status(mock_response(401))
|
|
88
87
|
assert_equal Ficon::BLOCKED, ficon.classify_response_status(mock_response(403))
|
|
89
88
|
assert_equal Ficon::BLOCKED, ficon.classify_response_status(mock_response(429))
|
|
90
|
-
|
|
89
|
+
|
|
91
90
|
# Test SICK status (5xx and others)
|
|
92
91
|
assert_equal Ficon::SICK, ficon.classify_response_status(mock_response(500))
|
|
93
92
|
assert_equal Ficon::SICK, ficon.classify_response_status(mock_response(502))
|
|
@@ -96,22 +95,79 @@ class FiconTest < Minitest::Test
|
|
|
96
95
|
end
|
|
97
96
|
|
|
98
97
|
def test_exception_status_classification
|
|
99
|
-
ficon = Ficon.new(
|
|
100
|
-
|
|
98
|
+
ficon = Ficon.new("https://example.com")
|
|
99
|
+
|
|
101
100
|
# Test DEAD status (DNS and resolution errors)
|
|
102
101
|
assert_equal Ficon::DEAD, ficon.classify_exception_status(SocketError.new)
|
|
103
102
|
assert_equal Ficon::DEAD, ficon.classify_exception_status(Resolv::ResolvError.new)
|
|
104
|
-
|
|
103
|
+
|
|
105
104
|
# Test SICK status (network and timeout errors)
|
|
106
105
|
assert_equal Ficon::SICK, ficon.classify_exception_status(Timeout::Error.new)
|
|
107
106
|
assert_equal Ficon::SICK, ficon.classify_exception_status(Errno::ECONNREFUSED.new)
|
|
108
107
|
assert_equal Ficon::SICK, ficon.classify_exception_status(OpenSSL::SSL::SSLError.new)
|
|
109
|
-
assert_equal Ficon::SICK, ficon.classify_exception_status(Net::HTTPError.new(
|
|
110
|
-
|
|
108
|
+
assert_equal Ficon::SICK, ficon.classify_exception_status(Net::HTTPError.new("error", nil))
|
|
109
|
+
|
|
111
110
|
# Test default to SICK for unknown exceptions
|
|
112
111
|
assert_equal Ficon::SICK, ficon.classify_exception_status(StandardError.new)
|
|
113
112
|
end
|
|
114
113
|
|
|
114
|
+
def test_http_to_https_fallback_conditions
|
|
115
|
+
# Test that localhost URLs are not converted to HTTPS
|
|
116
|
+
ficon_localhost = Ficon.allocate
|
|
117
|
+
uri_localhost = URI("http://localhost:3000/test")
|
|
118
|
+
|
|
119
|
+
# Mock the fetch_url method to simulate HTTP failure
|
|
120
|
+
def ficon_localhost.fetch_url(uri, redirect_limit = 5)
|
|
121
|
+
if uri.scheme == "http" && !uri.to_s.include?("://localhost") && !uri.host.match?(/^\d+\.\d+\.\d+\.\d+$/)
|
|
122
|
+
# This should not be reached for localhost
|
|
123
|
+
raise "Should not attempt HTTPS fallback for localhost"
|
|
124
|
+
end
|
|
125
|
+
nil
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# This should not raise an exception
|
|
129
|
+
result = ficon_localhost.send(:fetch_url, uri_localhost)
|
|
130
|
+
assert_nil result
|
|
131
|
+
|
|
132
|
+
# Test that IP addresses are not converted to HTTPS
|
|
133
|
+
ficon_ip = Ficon.allocate
|
|
134
|
+
uri_ip = URI("http://192.168.1.1/test")
|
|
135
|
+
|
|
136
|
+
def ficon_ip.fetch_url(uri, redirect_limit = 5)
|
|
137
|
+
if uri.scheme == "http" && !uri.to_s.include?("://localhost") && !uri.host.match?(/^\d+\.\d+\.\d+\.\d+$/)
|
|
138
|
+
# This should not be reached for IP addresses
|
|
139
|
+
raise "Should not attempt HTTPS fallback for IP addresses"
|
|
140
|
+
end
|
|
141
|
+
nil
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# This should not raise an exception
|
|
145
|
+
result = ficon_ip.send(:fetch_url, uri_ip)
|
|
146
|
+
assert_nil result
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def test_https_port_conversion
|
|
150
|
+
# Test that port 80 is converted to 443 when switching to HTTPS
|
|
151
|
+
http_uri = URI("http://example.com:80/test")
|
|
152
|
+
assert_equal 80, http_uri.port
|
|
153
|
+
|
|
154
|
+
https_uri = http_uri.dup
|
|
155
|
+
https_uri.scheme = "https"
|
|
156
|
+
https_uri.port = 443 if https_uri.port == 80
|
|
157
|
+
|
|
158
|
+
assert_equal "https", https_uri.scheme
|
|
159
|
+
assert_equal 443, https_uri.port
|
|
160
|
+
|
|
161
|
+
# Test that custom ports are preserved
|
|
162
|
+
http_custom_uri = URI("http://example.com:8080/test")
|
|
163
|
+
https_custom_uri = http_custom_uri.dup
|
|
164
|
+
https_custom_uri.scheme = "https"
|
|
165
|
+
https_custom_uri.port = 443 if https_custom_uri.port == 80
|
|
166
|
+
|
|
167
|
+
assert_equal "https", https_custom_uri.scheme
|
|
168
|
+
assert_equal 8080, https_custom_uri.port # Should remain unchanged
|
|
169
|
+
end
|
|
170
|
+
|
|
115
171
|
private
|
|
116
172
|
|
|
117
173
|
def mock_response(code)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ficon
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: '0.
|
|
4
|
+
version: '0.7'
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Dan Milne
|
|
@@ -159,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
159
159
|
- !ruby/object:Gem::Version
|
|
160
160
|
version: '0'
|
|
161
161
|
requirements: []
|
|
162
|
-
rubygems_version:
|
|
162
|
+
rubygems_version: 4.0.3
|
|
163
163
|
specification_version: 4
|
|
164
164
|
summary: Find website icons
|
|
165
165
|
test_files:
|