miteru 0.10.0 → 0.10.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/miteru/http_client.rb +18 -2
- data/lib/miteru/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a09a660f481a7f6c7000ad7943e0116a32629c137682bd5d944aef6a045aebd3
|
|
4
|
+
data.tar.gz: 402676cd08f35bdb0e5e2771f83a39fcde8c3ed4e5d8fc953176b85244828ebf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1b8d9cc4c82fdbc0c5d785a082f395e97d8e1a0550540170b900f894cc760ee31ae8755d5c3baed9b1c6a3681a88788e51ab8998bfcae80ca039ec7d3bdfd87a
|
|
7
|
+
data.tar.gz: 76b4aabd5fab653ef7d2133671ae6a770be2e9b8f52a0af36289bcf542a995e309ad723d7556c1de87d69c2c0eb8b90697d3c6cd260cdb33c3e0fc80c7d7f00b
|
data/lib/miteru/http_client.rb
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "http"
|
|
4
3
|
require "down/http"
|
|
4
|
+
require "http"
|
|
5
5
|
require "securerandom"
|
|
6
|
+
require "uri"
|
|
6
7
|
|
|
7
8
|
module Miteru
|
|
8
9
|
class HTTPClient
|
|
9
10
|
DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
|
|
11
|
+
URLSCAN_UA = "miteru/#{Miteru::VERSION}"
|
|
12
|
+
|
|
10
13
|
attr_reader :ssl_context
|
|
11
14
|
def initialize
|
|
12
15
|
ctx = OpenSSL::SSL::SSLContext.new
|
|
@@ -26,7 +29,11 @@ module Miteru
|
|
|
26
29
|
|
|
27
30
|
def get(url, options = {})
|
|
28
31
|
options = options.merge default_options
|
|
29
|
-
|
|
32
|
+
|
|
33
|
+
HTTP.follow
|
|
34
|
+
.timeout(write: 2, connect: 5, read: 10)
|
|
35
|
+
.headers(urlscan_url?(url) ? urlscan_headers : default_headers)
|
|
36
|
+
.get(url, options)
|
|
30
37
|
end
|
|
31
38
|
|
|
32
39
|
def self.get(url, options = {})
|
|
@@ -50,5 +57,14 @@ module Miteru
|
|
|
50
57
|
def default_options
|
|
51
58
|
{ ssl_context: ssl_context }
|
|
52
59
|
end
|
|
60
|
+
|
|
61
|
+
def urlscan_headers
|
|
62
|
+
{ user_agent: URLSCAN_UA }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def urlscan_url?(url)
|
|
66
|
+
uri = URI(url)
|
|
67
|
+
uri.hostname == "urlscan.io"
|
|
68
|
+
end
|
|
53
69
|
end
|
|
54
70
|
end
|
data/lib/miteru/version.rb
CHANGED