tweakphoeus 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/auto-tag.yaml +15 -0
- data/.github/workflows/main.yml +21 -0
- data/.github/workflows/on-pull-request.yaml +23 -0
- data/.github/workflows/on_new_tag.yaml +17 -0
- data/.github/workflows/version-forget-me-not.yaml +17 -0
- data/.rubocop.yml +6 -0
- data/CHANGELOG.md +24 -1
- data/Gemfile +2 -0
- data/README.md +5 -6
- data/Rakefile +5 -3
- data/bin/console +4 -3
- data/lib/tweakphoeus.rb +58 -142
- data/lib/tweakphoeus/cookie_jar.rb +79 -0
- data/lib/tweakphoeus/helpers.rb +10 -0
- data/lib/tweakphoeus/referer_list.rb +26 -0
- data/lib/tweakphoeus/user_agent.rb +17 -15
- data/lib/tweakphoeus/version.rb +3 -1
- data/tweakphoeus.gemspec +26 -26
- metadata +26 -29
- data/.travis.yml +0 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 29477b8f68a5920b4dfccb0bb6e3ac8cfcbc1598d5cf21707b128f11ee930538
|
|
4
|
+
data.tar.gz: fd0d5b8abf410c60ae8ca4938ac5fd85ec7165a4a0f40759635e6ac92c9db9d5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '0183ff90c46f024adc48de310cab5e076fe6a71b1334561f0c807c0f6afc3f946c76a096d3574db994eb1df5fa905c3be1be3cd5329c49436fa62095f5b7bd97'
|
|
7
|
+
data.tar.gz: 685b76dc8f0da6177761f9a264e63072282865f41f526bb9b7778252991af424743f65510fd798e2c7831c5e6620e3b5d4dc12d05be43d9e16ecc511218fdf9a
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: Ruby
|
|
2
|
+
|
|
3
|
+
on: [pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
steps:
|
|
9
|
+
- uses: actions/checkout@v2
|
|
10
|
+
- name: Set up Ruby
|
|
11
|
+
uses: ruby/setup-ruby@v1
|
|
12
|
+
with:
|
|
13
|
+
ruby-version: 2.7.4
|
|
14
|
+
bundler-cache: true
|
|
15
|
+
- name: Run the default task
|
|
16
|
+
run: bundle exec rake
|
|
17
|
+
services:
|
|
18
|
+
whoami:
|
|
19
|
+
image: containous/whoami
|
|
20
|
+
ports:
|
|
21
|
+
- 8080:80
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: Github on-pull-request Action
|
|
2
|
+
|
|
3
|
+
on: [pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
rubocop:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
steps:
|
|
9
|
+
- uses: actions/checkout@v2
|
|
10
|
+
- name: RuboCop Linter Action
|
|
11
|
+
uses: andrewmcodes/rubocop-linter-action@v3.3.0
|
|
12
|
+
env:
|
|
13
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
14
|
+
reek:
|
|
15
|
+
name: Reek
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v2
|
|
19
|
+
- uses: actions/setup-ruby@v1
|
|
20
|
+
with:
|
|
21
|
+
ruby-version: '2.7'
|
|
22
|
+
- run: gem install reek --no-doc
|
|
23
|
+
- run: reek .
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: On new tag
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- '*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish-new-version:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v2
|
|
13
|
+
- name: Publish gem
|
|
14
|
+
uses: dawidd6/action-publish-gem@v1
|
|
15
|
+
with:
|
|
16
|
+
api_key: ${{secrets.RUBYGEMS_API_KEY}}
|
|
17
|
+
github_token: ${{secrets.GITHUB_TOKEN}}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: Version Forget-Me-Not
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
- main
|
|
8
|
+
types: [opened, synchronize]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
semver-check:
|
|
12
|
+
runs-on: ubuntu-20.04
|
|
13
|
+
steps:
|
|
14
|
+
- uses: simplybusiness/version-forget-me-not@v2.1.0
|
|
15
|
+
env:
|
|
16
|
+
ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
17
|
+
VERSION_FILE_PATH: "lib/tweakphoeus/version.rb"
|
data/.rubocop.yml
ADDED
data/CHANGELOG.md
CHANGED
|
@@ -16,4 +16,27 @@
|
|
|
16
16
|
|
|
17
17
|
-- v0.4.1 - adding random User-Agent generator
|
|
18
18
|
|
|
19
|
-
-- v0.4.2 - adding ssl verification & deletion of active support methods
|
|
19
|
+
-- v0.4.2 - adding ssl verification & deletion of active support methods
|
|
20
|
+
|
|
21
|
+
-- v0.4.5 - pushing typhoeus version
|
|
22
|
+
|
|
23
|
+
-- v0.5.0 - Upgrading dependencies and revamping development environment
|
|
24
|
+
|
|
25
|
+
Bumping required ruby version to >=2.6.0
|
|
26
|
+
|
|
27
|
+
Upgrading Development dependencies:
|
|
28
|
+
- Bundler 2.2
|
|
29
|
+
- Rake 13.0
|
|
30
|
+
- Rspec 3.10
|
|
31
|
+
|
|
32
|
+
Swapping CI environment from CircleCI to Github Actions
|
|
33
|
+
|
|
34
|
+
Upgrading Typhoeus dependency to 1.4.0
|
|
35
|
+
|
|
36
|
+
-- v0.6.0 - Major refactor of Tweakphoeus::Client API
|
|
37
|
+
|
|
38
|
+
- New interface for Tweakphoeus::Client
|
|
39
|
+
- Initializer now accepts ssl_verifypeer and redirect parameters instead of having it on all HTTP verb methods (get, delete and post)
|
|
40
|
+
- ssl_verifypeer and redirect parameters removed from HTTP verb methods
|
|
41
|
+
- cookie_jar is not longer an Hash, it has been extracted to Tweakphoeus::CookieJar class
|
|
42
|
+
- refeer_list is not longer an Array, it has been extracted to Tweakphoeus::RefererList class
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
[](https://codeclimate.com/github/basestylo/Tweakphoeus)
|
|
1
|
+
[](https://rubygems.org/gems/tweakphoeus)
|
|
2
|
+
[](https://codeclimate.com/github/basestylo/Tweakphoeus/maintainability)
|
|
3
|
+
[](https://github.com/Naereen/StrapDown.js/blob/master/LICENSE)
|
|
5
4
|
[](https://codeclimate.com/github/basestylo/Tweakphoeus)
|
|
5
|
+
|
|
6
6
|
# Tweakphoeus
|
|
7
7
|
|
|
8
8
|
We usually describe this gem as 'typhoeus on steroids'. We add some browser features, for example cookies management and automation in tipical headers that browsers define in his HTTP stack.
|
|
9
9
|
|
|
10
|
-
We love scrapping and this gem was created for this.
|
|
10
|
+
We love scrapping and this gem was created for this.
|
|
11
11
|
|
|
12
12
|
From crazy developers to another crazy developers, Created in Bizneo.com and maintained in this small group of developers.
|
|
13
13
|
|
|
@@ -53,4 +53,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
|
|
|
53
53
|
## License
|
|
54
54
|
|
|
55
55
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
|
56
|
-
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
4
|
+
require 'bundler/setup'
|
|
5
|
+
require 'tweakphoeus'
|
|
5
6
|
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
|
@@ -10,5 +11,5 @@ require "tweakphoeus"
|
|
|
10
11
|
# require "pry"
|
|
11
12
|
# Pry.start
|
|
12
13
|
|
|
13
|
-
require
|
|
14
|
+
require 'irb'
|
|
14
15
|
IRB.start
|
data/lib/tweakphoeus.rb
CHANGED
|
@@ -1,87 +1,44 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'tweakphoeus/version'
|
|
4
|
+
require 'tweakphoeus/user_agent'
|
|
5
|
+
require 'typhoeus'
|
|
4
6
|
|
|
5
7
|
module Tweakphoeus
|
|
8
|
+
# Http client class
|
|
6
9
|
class Client
|
|
7
|
-
attr_accessor :cookie_jar
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def initialize()
|
|
11
|
-
@cookie_jar =
|
|
12
|
-
@
|
|
13
|
-
@
|
|
14
|
-
|
|
15
|
-
"Accept-Language" => "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3",
|
|
16
|
-
"Accept-Encoding" => "",
|
|
17
|
-
"Connection" => "keep-alive"
|
|
18
|
-
}
|
|
10
|
+
attr_accessor :cookie_jar, :referer_list, :base_headers, :redirect
|
|
11
|
+
attr_reader :ssl_verifypeer
|
|
12
|
+
|
|
13
|
+
def initialize(ua_systems: %i[linux mac], ssl_verifypeer: true, redirect: false)
|
|
14
|
+
@cookie_jar = Tweakphoeus::CookieJar.new
|
|
15
|
+
@referer_list = Tweakphoeus::RefererList.new
|
|
16
|
+
@redirect = redirect
|
|
17
|
+
@base_headers = build_base_headers(ua_systems)
|
|
19
18
|
@proxy = nil
|
|
20
19
|
@proxyuserpwd = nil
|
|
21
|
-
@ssl_verifypeer =
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def get(url, body: nil, params: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
|
|
25
|
-
set_referer_from_headers(headers)
|
|
26
|
-
http_request(url, body: body, params: params, headers: headers, redirect: redirect, method: :get, ssl_verifypeer: ssl_verifypeer)
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def delete(url, body: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
|
|
30
|
-
set_referer_from_headers(headers)
|
|
31
|
-
http_request(url, body: body, headers: headers, redirect: redirect, method: :delete, ssl_verifypeer: ssl_verifypeer)
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def post(url, body: nil, params: nil, headers: nil, redirect: false, ssl_verifypeer: @ssl_verifypeer)
|
|
35
|
-
set_referer_from_headers(headers)
|
|
36
|
-
http_request(url, body: body, params: nil, headers: headers, redirect: redirect, method: :post, ssl_verifypeer: ssl_verifypeer)
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
def get_hide_inputs response
|
|
40
|
-
#TODO
|
|
20
|
+
@ssl_verifypeer = ssl_verifypeer
|
|
41
21
|
end
|
|
42
22
|
|
|
43
|
-
def
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
@cookie_jar[domain][key] = value
|
|
23
|
+
def get(url, body: nil, params: nil, headers: nil)
|
|
24
|
+
referer_from_headers(headers)
|
|
25
|
+
http_request(url, body: body, params: params, headers: headers, method: :get)
|
|
47
26
|
end
|
|
48
27
|
|
|
49
|
-
def
|
|
50
|
-
|
|
28
|
+
def delete(url, body: nil, headers: nil)
|
|
29
|
+
referer_from_headers(headers)
|
|
30
|
+
http_request(url, body: body, headers: headers, method: :delete)
|
|
51
31
|
end
|
|
52
32
|
|
|
53
|
-
def
|
|
54
|
-
|
|
55
|
-
url
|
|
33
|
+
def post(url, body: nil, headers: nil)
|
|
34
|
+
referer_from_headers(headers)
|
|
35
|
+
http_request(url, body: body, headers: headers, method: :post)
|
|
56
36
|
end
|
|
57
37
|
|
|
58
|
-
def
|
|
59
|
-
|
|
60
|
-
end
|
|
38
|
+
def set_proxy(url, auth)
|
|
39
|
+
return unless auth
|
|
61
40
|
|
|
62
|
-
|
|
63
|
-
@referer.last
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def cookie_string(url, headers={})
|
|
67
|
-
domain = get_domain(url)
|
|
68
|
-
headers ||= {}
|
|
69
|
-
cookies = parse_cookie(headers["Cookie"])
|
|
70
|
-
|
|
71
|
-
while domain != ""
|
|
72
|
-
if @cookie_jar[domain]
|
|
73
|
-
@cookie_jar[domain].each do |key, value|
|
|
74
|
-
cookies[key] ||= value
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
domain = domain.gsub(/^([^\.]+\.?)/, '')
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
cookies.map{ |key, value| "#{key}=#{value}" }.join('; ')
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def set_proxy(url, auth = nil)
|
|
84
|
-
@proxyuserpwd = "#{auth[:user]}:#{auth[:password]}" if auth.is_a?(Hash)
|
|
41
|
+
@proxyuserpwd = "#{auth[:user]}:#{auth[:password]}"
|
|
85
42
|
@proxy = url
|
|
86
43
|
end
|
|
87
44
|
|
|
@@ -92,94 +49,53 @@ module Tweakphoeus
|
|
|
92
49
|
|
|
93
50
|
private
|
|
94
51
|
|
|
95
|
-
def http_request(url, body: nil, params: nil, headers: nil,
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
request_headers["Referer"] = get_referer
|
|
99
|
-
response = Typhoeus.send(method, url, body: body, params: params, headers: request_headers, proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
|
|
100
|
-
obtain_cookies(response)
|
|
101
|
-
set_referer(url) if method != :post
|
|
102
|
-
if redirect && has_redirect?(response)
|
|
103
|
-
if response.code != 307
|
|
104
|
-
method = :get
|
|
105
|
-
body = nil
|
|
106
|
-
end
|
|
107
|
-
response = http_request(redirect_url(response),
|
|
108
|
-
body: body,
|
|
109
|
-
headers: headers,
|
|
110
|
-
redirect: redirect,
|
|
111
|
-
method: method,
|
|
112
|
-
ssl_verifypeer: ssl_verifypeer)
|
|
113
|
-
end
|
|
114
|
-
response
|
|
115
|
-
end
|
|
52
|
+
def http_request(url, body: nil, params: nil, headers: nil, method: :get)
|
|
53
|
+
response = Typhoeus.send(method, url, body: body, params: params, headers: build_request_headers(headers),
|
|
54
|
+
proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
|
|
116
55
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
end
|
|
56
|
+
@cookie_jar.obtain_cookies(response)
|
|
57
|
+
@referer_list.push_referer(url) if method != :post
|
|
120
58
|
|
|
121
|
-
|
|
122
|
-
set_cookies_field = response.headers["Set-Cookie"]
|
|
123
|
-
return if set_cookies_field.nil?
|
|
124
|
-
if set_cookies_field.is_a?(String)
|
|
125
|
-
set_cookies_field = [set_cookies_field]
|
|
126
|
-
end
|
|
59
|
+
return process_redirect(response) if redirect && redirect?(response)
|
|
127
60
|
|
|
128
|
-
|
|
129
|
-
key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..-1]
|
|
130
|
-
domain = cookie.match(/Domain=([^;]+)/)
|
|
131
|
-
|
|
132
|
-
if domain.nil?
|
|
133
|
-
domain = get_domain(response.request.url)
|
|
134
|
-
else
|
|
135
|
-
domain = domain[1].gsub(/^\./,'')
|
|
136
|
-
end
|
|
137
|
-
|
|
138
|
-
value = value.split(';').first
|
|
139
|
-
if value != "\"\""
|
|
140
|
-
@cookie_jar[domain] ||= {}
|
|
141
|
-
@cookie_jar[domain][key] = value
|
|
142
|
-
else
|
|
143
|
-
@cookie_jar[domain] ||= {}
|
|
144
|
-
@cookie_jar[domain].delete(key)
|
|
145
|
-
end
|
|
146
|
-
end
|
|
61
|
+
response
|
|
147
62
|
end
|
|
148
63
|
|
|
149
|
-
def
|
|
150
|
-
|
|
64
|
+
def build_request_headers(headers)
|
|
65
|
+
request_headers = merge_default_headers(headers)
|
|
66
|
+
request_headers['Cookie'] = @cookie_jar.cookie_string(url, headers)
|
|
67
|
+
request_headers['Referer'] = @referer_list.last_referer
|
|
68
|
+
end
|
|
151
69
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
cookies[key.strip] = value if value && !["Domain","Path","domain","path"].include?(key)
|
|
157
|
-
end
|
|
70
|
+
def process_redirect(response)
|
|
71
|
+
if response.code != 307
|
|
72
|
+
method = :get
|
|
73
|
+
body = nil
|
|
158
74
|
end
|
|
159
75
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
def has_redirect? response
|
|
164
|
-
!redirect_url(response).nil?
|
|
76
|
+
http_request(redirect_url(response), body: body, headers: headers,
|
|
77
|
+
redirect: redirect, method: method, ssl_verifypeer: ssl_verifypeer)
|
|
165
78
|
end
|
|
166
79
|
|
|
167
|
-
def
|
|
168
|
-
|
|
80
|
+
def build_base_headers(ua_systems)
|
|
81
|
+
{
|
|
82
|
+
'User-Agent' => UserAgent.random(systems: ua_systems),
|
|
83
|
+
'Accept-Language' => 'es-ES,es;q=0.9,en;q=0.8',
|
|
84
|
+
'Accept-Encoding' => '',
|
|
85
|
+
'Connection' => 'keep-alive'
|
|
86
|
+
}
|
|
169
87
|
end
|
|
170
88
|
|
|
171
|
-
def
|
|
172
|
-
|
|
89
|
+
def merge_default_headers(headers)
|
|
90
|
+
headers ? @base_headers.merge(headers) : @base_headers
|
|
173
91
|
end
|
|
174
92
|
|
|
175
|
-
def
|
|
176
|
-
|
|
177
|
-
@referer.last.replace headers["Referer"]
|
|
178
|
-
end
|
|
93
|
+
def redirect?(response)
|
|
94
|
+
!redirect_url(response).nil?
|
|
179
95
|
end
|
|
180
96
|
|
|
181
|
-
def
|
|
182
|
-
|
|
97
|
+
def redirect_url(response)
|
|
98
|
+
response.headers['Location']
|
|
183
99
|
end
|
|
184
100
|
end
|
|
185
101
|
end
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'tweakphoeus/helpers'
|
|
4
|
+
|
|
5
|
+
module Tweakphoeus
|
|
6
|
+
# Cookie manager
|
|
7
|
+
class CookieJar
|
|
8
|
+
include Tweakphoeus::Helpers
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@cookie_jar = {}
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def purge_bad_cookies(cookies)
|
|
15
|
+
cookies.reject { |cookie| cookie.first.last == '""' }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def cookie_string(url, headers = {})
|
|
19
|
+
domain = get_domain(url)
|
|
20
|
+
headers ||= {}
|
|
21
|
+
cookies = parse_cookie(headers['Cookie'])
|
|
22
|
+
|
|
23
|
+
while domain != ''
|
|
24
|
+
@cookie_jar[domain]&.each do |key, value|
|
|
25
|
+
cookies[key] ||= value
|
|
26
|
+
end
|
|
27
|
+
domain = domain.gsub(/^([^.]+\.?)/, '')
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
cookies.map { |key, value| "#{key}=#{value}" }.join('; ')
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def obtain_cookies(response)
|
|
34
|
+
set_cookies_field = [response.headers['Set-Cookie']].compact
|
|
35
|
+
|
|
36
|
+
set_cookies_field.each do |cookie|
|
|
37
|
+
key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..]
|
|
38
|
+
domain = cookie.match(/Domain=([^;]+)/)&.at(1)&.gsub(/^\./, '')
|
|
39
|
+
|
|
40
|
+
domain = get_domain(response.request.url) if domain.nil
|
|
41
|
+
set_cookie(domain, key, value)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def add_cookies(host, key, value)
|
|
46
|
+
domain = get_domain(host)
|
|
47
|
+
@cookie_jar[domain] ||= {}
|
|
48
|
+
@cookie_jar[domain][key] = value
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def set_cookie(domain, key, value)
|
|
54
|
+
@cookie_jar[domain] ||= {}
|
|
55
|
+
|
|
56
|
+
value = value.split(';').first
|
|
57
|
+
|
|
58
|
+
if value == '""'
|
|
59
|
+
@cookie_jar[domain].delete(key)
|
|
60
|
+
else
|
|
61
|
+
@cookie_jar[domain][key] = value
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def parse_cookie(string)
|
|
66
|
+
cookies = {}
|
|
67
|
+
|
|
68
|
+
if string.is_a?(String)
|
|
69
|
+
string.split(';').each do |part|
|
|
70
|
+
key, value = part.split('=')
|
|
71
|
+
key.strip!
|
|
72
|
+
cookies[key.strip] = value if value && !%w[Domain Path domain path].include?(key)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
cookies
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tweakphoeus
|
|
4
|
+
# RefererList class handle referer logic for Referer header in requests
|
|
5
|
+
class RefererList
|
|
6
|
+
def initialize
|
|
7
|
+
@referer = []
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def push_referer(url)
|
|
11
|
+
@referer << url
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def pop_referer
|
|
15
|
+
@referer.pop || ''
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def last_referer
|
|
19
|
+
@referer.last
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def referer_from_headers(headers)
|
|
23
|
+
@referer.last.replace(headers['Referer']) if headers && headers['Referer'].is_a?(String)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -1,17 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Tweakphoeus
|
|
4
|
+
# User agent data generator
|
|
2
5
|
class UserAgent
|
|
3
6
|
OS_DATA = {
|
|
4
|
-
types: [
|
|
5
|
-
linux_distro: [
|
|
6
|
-
arch: [
|
|
7
|
-
win_arch: [
|
|
8
|
-
}
|
|
7
|
+
types: %i[win linux mac],
|
|
8
|
+
linux_distro: ['Ubuntu', 'Debian', 'Fedora', 'gNewSense', 'Linux Mint', 'OpenSUSE', 'Mandriva'],
|
|
9
|
+
arch: %w[x86 x86_64],
|
|
10
|
+
win_arch: ['Win64; x64', 'WOW64']
|
|
11
|
+
}.freeze
|
|
9
12
|
|
|
10
13
|
class << self
|
|
11
14
|
def random(systems: OS_DATA[:types])
|
|
12
15
|
user_agent = "Mozilla/5.0 (#{os[systems.sample]}) #{browser}"
|
|
13
|
-
firefox_version = user_agent.match(/
|
|
14
|
-
user_agent.gsub!(')', "; rv:#{firefox_version})") if firefox_version
|
|
16
|
+
firefox_version = user_agent.match(%r{Firefox/([0-9.]+)})
|
|
17
|
+
user_agent.gsub!(')', "; rv:#{firefox_version[1]})") if firefox_version.is_a?(MatchData)
|
|
15
18
|
user_agent
|
|
16
19
|
end
|
|
17
20
|
|
|
@@ -19,19 +22,18 @@ module Tweakphoeus
|
|
|
19
22
|
|
|
20
23
|
def os
|
|
21
24
|
{
|
|
22
|
-
win: "Windows NT #{%w
|
|
23
|
-
linux: "X11; #{[
|
|
24
|
-
mac: "Macintosh; Intel Mac OS X 10_#{Random.rand(
|
|
25
|
+
win: "Windows NT #{"#{%w[5 6 7 8 10].sample}.#{%w[0 1].sample}"}",
|
|
26
|
+
linux: "X11; #{['', "#{OS_DATA[:linux_distro].sample}; "].sample}Linux #{OS_DATA[:arch].sample}",
|
|
27
|
+
mac: "Macintosh; Intel Mac OS X 10_#{Random.rand(11..13)}_#{Random.rand(0..3)}"
|
|
25
28
|
}
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
def browser
|
|
29
|
-
chrome_version = "Chrome
|
|
30
|
-
|
|
31
|
-
["Gecko/20100101 Firefox/#{Random.rand(
|
|
32
|
+
chrome_version = "Chrome/#{Random.rand(90..92)}.0.#{Random.rand(1000..4000)}.#{Random.rand(100..900)}"
|
|
33
|
+
webkit603 = "603.#{Random.rand(0..9)}.#{Random.rand(0..9)}"
|
|
34
|
+
["Gecko/20100101 Firefox/#{Random.rand(82..90)}.0",
|
|
32
35
|
"AppleWebKit/537.36 (KHTML, like Gecko) #{chrome_version} Safari/537.36",
|
|
33
|
-
"AppleWebKit/#{
|
|
34
|
-
].sample
|
|
36
|
+
"AppleWebKit/#{webkit603} (KHTML, like Gecko) Version/10.1 #{chrome_version} Safari/#{webkit603}"].sample
|
|
35
37
|
end
|
|
36
38
|
end
|
|
37
39
|
end
|
data/lib/tweakphoeus/version.rb
CHANGED
data/tweakphoeus.gemspec
CHANGED
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
require 'tweakphoeus/version'
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'lib/tweakphoeus/version'
|
|
5
4
|
|
|
6
5
|
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name =
|
|
6
|
+
spec.name = 'tweakphoeus'
|
|
8
7
|
spec.version = Tweakphoeus::VERSION
|
|
9
|
-
spec.authors = [
|
|
10
|
-
spec.email = [
|
|
8
|
+
spec.authors = ['David Martin Garcia']
|
|
9
|
+
spec.email = ['davidmartingarcia@gmail.com']
|
|
11
10
|
|
|
12
|
-
spec.summary =
|
|
13
|
-
spec.description =
|
|
14
|
-
spec.homepage =
|
|
15
|
-
spec.license =
|
|
11
|
+
spec.summary = 'Typhoeus on steroids.'
|
|
12
|
+
spec.description = 'Typhoeus wrapper with some extras.'
|
|
13
|
+
spec.homepage = 'https://github.com/basestylo/Tweakphoeus/'
|
|
14
|
+
spec.license = 'MIT'
|
|
15
|
+
spec.required_ruby_version = '>= 2.6.0'
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
|
21
|
-
else
|
|
22
|
-
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
|
23
|
-
end
|
|
17
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
|
18
|
+
spec.metadata['source_code_uri'] = 'https://github.com/dmartingarcia/Tweakphoeus'
|
|
19
|
+
spec.metadata['changelog_uri'] = 'https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md'
|
|
24
20
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
spec.
|
|
28
|
-
|
|
21
|
+
# Specify which files should be added to the gem when it is released.
|
|
22
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
23
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
24
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
25
|
+
end
|
|
26
|
+
spec.bindir = 'exe'
|
|
27
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
28
|
+
spec.require_paths = ['lib']
|
|
29
29
|
|
|
30
|
-
spec.add_dependency 'typhoeus', '~> 1.
|
|
31
|
-
spec.add_development_dependency
|
|
32
|
-
spec.add_development_dependency
|
|
33
|
-
spec.add_development_dependency
|
|
30
|
+
spec.add_dependency 'typhoeus', '~> 1.4'
|
|
31
|
+
spec.add_development_dependency 'bundler', '~> 2.2'
|
|
32
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
|
33
|
+
spec.add_development_dependency 'rspec', '~> 3.10'
|
|
34
34
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tweakphoeus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David Martin Garcia
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2021-08-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: typhoeus
|
|
@@ -16,78 +16,71 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1.
|
|
20
|
-
- - ">="
|
|
21
|
-
- !ruby/object:Gem::Version
|
|
22
|
-
version: 1.0.1
|
|
19
|
+
version: '1.4'
|
|
23
20
|
type: :runtime
|
|
24
21
|
prerelease: false
|
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
26
23
|
requirements:
|
|
27
24
|
- - "~>"
|
|
28
25
|
- !ruby/object:Gem::Version
|
|
29
|
-
version: '1.
|
|
30
|
-
- - ">="
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: 1.0.1
|
|
26
|
+
version: '1.4'
|
|
33
27
|
- !ruby/object:Gem::Dependency
|
|
34
28
|
name: bundler
|
|
35
29
|
requirement: !ruby/object:Gem::Requirement
|
|
36
30
|
requirements:
|
|
37
31
|
- - "~>"
|
|
38
32
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
33
|
+
version: '2.2'
|
|
40
34
|
type: :development
|
|
41
35
|
prerelease: false
|
|
42
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
43
37
|
requirements:
|
|
44
38
|
- - "~>"
|
|
45
39
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '
|
|
40
|
+
version: '2.2'
|
|
47
41
|
- !ruby/object:Gem::Dependency
|
|
48
42
|
name: rake
|
|
49
43
|
requirement: !ruby/object:Gem::Requirement
|
|
50
44
|
requirements:
|
|
51
45
|
- - "~>"
|
|
52
46
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '
|
|
47
|
+
version: '13.0'
|
|
54
48
|
type: :development
|
|
55
49
|
prerelease: false
|
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
57
51
|
requirements:
|
|
58
52
|
- - "~>"
|
|
59
53
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '
|
|
54
|
+
version: '13.0'
|
|
61
55
|
- !ruby/object:Gem::Dependency
|
|
62
56
|
name: rspec
|
|
63
57
|
requirement: !ruby/object:Gem::Requirement
|
|
64
58
|
requirements:
|
|
65
59
|
- - "~>"
|
|
66
60
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '3.
|
|
68
|
-
- - ">"
|
|
69
|
-
- !ruby/object:Gem::Version
|
|
70
|
-
version: 3.0.0
|
|
61
|
+
version: '3.10'
|
|
71
62
|
type: :development
|
|
72
63
|
prerelease: false
|
|
73
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
74
65
|
requirements:
|
|
75
66
|
- - "~>"
|
|
76
67
|
- !ruby/object:Gem::Version
|
|
77
|
-
version: '3.
|
|
78
|
-
|
|
79
|
-
- !ruby/object:Gem::Version
|
|
80
|
-
version: 3.0.0
|
|
81
|
-
description: 'Typhoeus wrapper with new pretty things: cookie handler for example.'
|
|
68
|
+
version: '3.10'
|
|
69
|
+
description: Typhoeus wrapper with some extras.
|
|
82
70
|
email:
|
|
83
|
-
-
|
|
71
|
+
- davidmartingarcia@gmail.com
|
|
84
72
|
executables: []
|
|
85
73
|
extensions: []
|
|
86
74
|
extra_rdoc_files: []
|
|
87
75
|
files:
|
|
76
|
+
- ".github/workflows/auto-tag.yaml"
|
|
77
|
+
- ".github/workflows/main.yml"
|
|
78
|
+
- ".github/workflows/on-pull-request.yaml"
|
|
79
|
+
- ".github/workflows/on_new_tag.yaml"
|
|
80
|
+
- ".github/workflows/version-forget-me-not.yaml"
|
|
88
81
|
- ".gitignore"
|
|
89
82
|
- ".rspec"
|
|
90
|
-
- ".
|
|
83
|
+
- ".rubocop.yml"
|
|
91
84
|
- CHANGELOG.md
|
|
92
85
|
- CODE_OF_CONDUCT.md
|
|
93
86
|
- Gemfile
|
|
@@ -97,6 +90,9 @@ files:
|
|
|
97
90
|
- bin/console
|
|
98
91
|
- bin/setup
|
|
99
92
|
- lib/tweakphoeus.rb
|
|
93
|
+
- lib/tweakphoeus/cookie_jar.rb
|
|
94
|
+
- lib/tweakphoeus/helpers.rb
|
|
95
|
+
- lib/tweakphoeus/referer_list.rb
|
|
100
96
|
- lib/tweakphoeus/user_agent.rb
|
|
101
97
|
- lib/tweakphoeus/version.rb
|
|
102
98
|
- tweakphoeus.gemspec
|
|
@@ -104,7 +100,9 @@ homepage: https://github.com/basestylo/Tweakphoeus/
|
|
|
104
100
|
licenses:
|
|
105
101
|
- MIT
|
|
106
102
|
metadata:
|
|
107
|
-
|
|
103
|
+
homepage_uri: https://github.com/basestylo/Tweakphoeus/
|
|
104
|
+
source_code_uri: https://github.com/dmartingarcia/Tweakphoeus
|
|
105
|
+
changelog_uri: https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md
|
|
108
106
|
post_install_message:
|
|
109
107
|
rdoc_options: []
|
|
110
108
|
require_paths:
|
|
@@ -113,15 +111,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
113
111
|
requirements:
|
|
114
112
|
- - ">="
|
|
115
113
|
- !ruby/object:Gem::Version
|
|
116
|
-
version:
|
|
114
|
+
version: 2.6.0
|
|
117
115
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
116
|
requirements:
|
|
119
117
|
- - ">="
|
|
120
118
|
- !ruby/object:Gem::Version
|
|
121
119
|
version: '0'
|
|
122
120
|
requirements: []
|
|
123
|
-
|
|
124
|
-
rubygems_version: 2.6.12
|
|
121
|
+
rubygems_version: 3.1.2
|
|
125
122
|
signing_key:
|
|
126
123
|
specification_version: 4
|
|
127
124
|
summary: Typhoeus on steroids.
|
data/.travis.yml
DELETED