tweakphoeus 0.4.5 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/auto-tag.yaml +15 -0
- data/.github/workflows/main.yml +24 -0
- data/.github/workflows/on-pull-request.yaml +23 -0
- data/.github/workflows/on_new_tag.yaml +17 -0
- data/.github/workflows/version-forget-me-not.yaml +17 -0
- data/.rubocop.yml +6 -0
- data/CHANGELOG.md +26 -0
- data/Gemfile +2 -0
- data/README.md +5 -6
- data/Rakefile +5 -3
- data/bin/console +4 -3
- data/lib/tweakphoeus/cookie_jar.rb +79 -0
- data/lib/tweakphoeus/helpers.rb +10 -0
- data/lib/tweakphoeus/referer_list.rb +26 -0
- data/lib/tweakphoeus/user_agent.rb +16 -15
- data/lib/tweakphoeus/version.rb +3 -1
- data/lib/tweakphoeus.rb +60 -142
- data/tweakphoeus.gemspec +26 -26
- metadata +26 -29
- data/.travis.yml +0 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c2fc171c1f446511f0c9ad337b4885e9ffc8cd0fe28ea3089875cf5a15a7fb25
|
|
4
|
+
data.tar.gz: 59583e9143549eb48d50b58e01e84155082e71c573191a3e4ce39f791ddd5e32
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 746e43e153431a56c07a7ec5c51ffa1a90f4e800c1018cb3792214877e10edc2e0aa3278a6290a43231192da9045170615bf6f32b1e0d521bc7ccb25d5e8bd28
|
|
7
|
+
data.tar.gz: '043856a637bbfb0a9fb5cdfd27f1bcba6ea8e132f3fdff9e0ce119f32578d042b039668f855383570c0b7ff02553c6b087b2506a632fd56bfa4fcd8500e2fa96'
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: Ruby
|
|
2
|
+
|
|
3
|
+
on: [pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
strategy:
|
|
9
|
+
matrix:
|
|
10
|
+
ruby: [ '2.6', '2.7', '3.0' ]
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v2
|
|
13
|
+
- name: Set up Ruby ${{ matrix.ruby }}
|
|
14
|
+
uses: ruby/setup-ruby@v1
|
|
15
|
+
with:
|
|
16
|
+
ruby-version: ${{ matrix.ruby }}
|
|
17
|
+
bundler-cache: true
|
|
18
|
+
- name: Run the default task
|
|
19
|
+
run: bundle exec rake
|
|
20
|
+
services:
|
|
21
|
+
whoami:
|
|
22
|
+
image: containous/whoami
|
|
23
|
+
ports:
|
|
24
|
+
- 8080:80
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: Github on-pull-request Action
|
|
2
|
+
|
|
3
|
+
on: [pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
rubocop:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
steps:
|
|
9
|
+
- uses: actions/checkout@v2
|
|
10
|
+
- name: RuboCop Linter Action
|
|
11
|
+
uses: andrewmcodes/rubocop-linter-action@v3.3.0
|
|
12
|
+
env:
|
|
13
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
14
|
+
reek:
|
|
15
|
+
name: Reek
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v2
|
|
19
|
+
- uses: actions/setup-ruby@v1
|
|
20
|
+
with:
|
|
21
|
+
ruby-version: '2.7'
|
|
22
|
+
- run: gem install reek --no-doc
|
|
23
|
+
- run: reek .
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: On new tag
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- '*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish-new-version:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v2
|
|
13
|
+
- name: Publish gem
|
|
14
|
+
uses: dawidd6/action-publish-gem@v1
|
|
15
|
+
with:
|
|
16
|
+
api_key: ${{secrets.RUBYGEMS_API_KEY}}
|
|
17
|
+
github_token: ${{secrets.GITHUB_TOKEN}}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
name: Version Forget-Me-Not
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
- main
|
|
8
|
+
types: [opened, synchronize]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
semver-check:
|
|
12
|
+
runs-on: ubuntu-20.04
|
|
13
|
+
steps:
|
|
14
|
+
- uses: simplybusiness/version-forget-me-not@v2.1.0
|
|
15
|
+
env:
|
|
16
|
+
ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
17
|
+
VERSION_FILE_PATH: "lib/tweakphoeus/version.rb"
|
data/.rubocop.yml
ADDED
data/CHANGELOG.md
CHANGED
|
@@ -19,3 +19,29 @@
|
|
|
19
19
|
-- v0.4.2 - adding ssl verification & deletion of active support methods
|
|
20
20
|
|
|
21
21
|
-- v0.4.5 - pushing typhoeus version
|
|
22
|
+
|
|
23
|
+
-- v0.5.0 - Upgrading dependencies and revamping development environment
|
|
24
|
+
|
|
25
|
+
Bumping required ruby version to >=2.6.0
|
|
26
|
+
|
|
27
|
+
Upgrading Development dependencies:
|
|
28
|
+
- Bundler 2.2
|
|
29
|
+
- Rake 13.0
|
|
30
|
+
- Rspec 3.10
|
|
31
|
+
|
|
32
|
+
Swapping CI environment from CircleCI to Github Actions
|
|
33
|
+
|
|
34
|
+
Upgrading Typhoeus dependency to 1.4.0
|
|
35
|
+
|
|
36
|
+
-- v0.6.0 - Major refactor of Tweakphoeus::Client API
|
|
37
|
+
|
|
38
|
+
- New interface for Tweakphoeus::Client
|
|
39
|
+
- Initializer now accepts ssl_verifypeer and redirect parameters instead of having it on all HTTP verb methods (get, delete and post)
|
|
40
|
+
- ssl_verifypeer and redirect parameters removed from HTTP verb methods
|
|
41
|
+
- cookie_jar is not longer an Hash, it has been extracted to Tweakphoeus::CookieJar class
|
|
42
|
+
- refeer_list is not longer an Array, it has been extracted to Tweakphoeus::RefererList class
|
|
43
|
+
|
|
44
|
+
-- v0.6.1 - Fixing issues with the modification of a frozen string
|
|
45
|
+
- Adds missing requires in the mail `.rb` file
|
|
46
|
+
- Solves an issue with a modification on a frozen string
|
|
47
|
+
> /Users/davidmartingarcia/.rbenv/versions/2.7.4/lib/ruby/gems/2.7.0/gems/tweakphoeus-0.6.0/lib/tweakphoeus/user_agent.rb:17:in `gsub!': can't modify frozen String: "Mozilla/5.0 (X11; gNewSense; Linux x86) Gecko/20100101 Firefox/84.0" (FrozenError)
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
[](https://codeclimate.com/github/basestylo/Tweakphoeus)
|
|
1
|
+
[](https://rubygems.org/gems/tweakphoeus)
|
|
2
|
+
[](https://codeclimate.com/github/basestylo/Tweakphoeus/maintainability)
|
|
3
|
+
[](https://github.com/Naereen/StrapDown.js/blob/master/LICENSE)
|
|
5
4
|
[](https://codeclimate.com/github/basestylo/Tweakphoeus)
|
|
5
|
+
|
|
6
6
|
# Tweakphoeus
|
|
7
7
|
|
|
8
8
|
We usually describe this gem as 'typhoeus on steroids'. We add some browser features, for example cookies management and automation in tipical headers that browsers define in his HTTP stack.
|
|
9
9
|
|
|
10
|
-
We love scrapping and this gem was created for this.
|
|
10
|
+
We love scrapping and this gem was created for this.
|
|
11
11
|
|
|
12
12
|
From crazy developers to another crazy developers, Created in Bizneo.com and maintained in this small group of developers.
|
|
13
13
|
|
|
@@ -53,4 +53,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
|
|
|
53
53
|
## License
|
|
54
54
|
|
|
55
55
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
|
56
|
-
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
4
|
+
require 'bundler/setup'
|
|
5
|
+
require 'tweakphoeus'
|
|
5
6
|
|
|
6
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
8
|
# with your gem easier. You can also use a different console, if you like.
|
|
@@ -10,5 +11,5 @@ require "tweakphoeus"
|
|
|
10
11
|
# require "pry"
|
|
11
12
|
# Pry.start
|
|
12
13
|
|
|
13
|
-
require
|
|
14
|
+
require 'irb'
|
|
14
15
|
IRB.start
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'tweakphoeus/helpers'
|
|
4
|
+
|
|
5
|
+
module Tweakphoeus
|
|
6
|
+
# Cookie manager
|
|
7
|
+
class CookieJar
|
|
8
|
+
include Tweakphoeus::Helpers
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@cookie_jar = {}
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def purge_bad_cookies(cookies)
|
|
15
|
+
cookies.reject { |cookie| cookie.first.last == '""' }
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def cookie_string(url, headers = {})
|
|
19
|
+
domain = get_domain(url)
|
|
20
|
+
headers ||= {}
|
|
21
|
+
cookies = parse_cookie(headers['Cookie'])
|
|
22
|
+
|
|
23
|
+
while domain != ''
|
|
24
|
+
@cookie_jar[domain]&.each do |key, value|
|
|
25
|
+
cookies[key] ||= value
|
|
26
|
+
end
|
|
27
|
+
domain = domain.gsub(/^([^.]+\.?)/, '')
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
cookies.map { |key, value| "#{key}=#{value}" }.join('; ')
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def obtain_cookies(response)
|
|
34
|
+
set_cookies_field = [response.headers['Set-Cookie']].compact
|
|
35
|
+
|
|
36
|
+
set_cookies_field.each do |cookie|
|
|
37
|
+
key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..]
|
|
38
|
+
domain = cookie.match(/Domain=([^;]+)/)&.at(1)&.gsub(/^\./, '')
|
|
39
|
+
|
|
40
|
+
domain = get_domain(response.request.url) if domain.nil
|
|
41
|
+
set_cookie(domain, key, value)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def add_cookies(host, key, value)
|
|
46
|
+
domain = get_domain(host)
|
|
47
|
+
@cookie_jar[domain] ||= {}
|
|
48
|
+
@cookie_jar[domain][key] = value
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def set_cookie(domain, key, value)
|
|
54
|
+
@cookie_jar[domain] ||= {}
|
|
55
|
+
|
|
56
|
+
value = value.split(';').first
|
|
57
|
+
|
|
58
|
+
if value == '""'
|
|
59
|
+
@cookie_jar[domain].delete(key)
|
|
60
|
+
else
|
|
61
|
+
@cookie_jar[domain][key] = value
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def parse_cookie(string)
|
|
66
|
+
cookies = {}
|
|
67
|
+
|
|
68
|
+
if string.is_a?(String)
|
|
69
|
+
string.split(';').each do |part|
|
|
70
|
+
key, value = part.split('=')
|
|
71
|
+
key.strip!
|
|
72
|
+
cookies[key.strip] = value if value && !%w[Domain Path domain path].include?(key)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
cookies
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tweakphoeus
|
|
4
|
+
# RefererList class handle referer logic for Referer header in requests
|
|
5
|
+
class RefererList
|
|
6
|
+
def initialize
|
|
7
|
+
@referer = []
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def push_referer(url)
|
|
11
|
+
@referer << url
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def pop_referer
|
|
15
|
+
@referer.pop || ''
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def last_referer
|
|
19
|
+
@referer.last
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def referer_from_headers(headers)
|
|
23
|
+
@referer.last.replace(headers['Referer']) if headers && headers['Referer'].is_a?(String)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -1,37 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Tweakphoeus
|
|
4
|
+
# User agent data generator
|
|
2
5
|
class UserAgent
|
|
3
6
|
OS_DATA = {
|
|
4
|
-
types: [
|
|
5
|
-
linux_distro: [
|
|
6
|
-
arch: [
|
|
7
|
-
win_arch: [
|
|
8
|
-
}
|
|
7
|
+
types: %i[win linux mac],
|
|
8
|
+
linux_distro: ['Ubuntu', 'Debian', 'Fedora', 'gNewSense', 'Linux Mint', 'OpenSUSE', 'Mandriva'],
|
|
9
|
+
arch: %w[x86 x86_64],
|
|
10
|
+
win_arch: ['Win64; x64', 'WOW64']
|
|
11
|
+
}.freeze
|
|
9
12
|
|
|
10
13
|
class << self
|
|
11
14
|
def random(systems: OS_DATA[:types])
|
|
12
15
|
user_agent = "Mozilla/5.0 (#{os[systems.sample]}) #{browser}"
|
|
13
|
-
firefox_version = user_agent.match(/
|
|
14
|
-
user_agent.gsub
|
|
15
|
-
user_agent
|
|
16
|
+
firefox_version = user_agent.match(%r{Firefox/([0-9.]+)})
|
|
17
|
+
user_agent.gsub(')', "; rv:#{firefox_version[1]})") if firefox_version.is_a?(MatchData)
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
private
|
|
19
21
|
|
|
20
22
|
def os
|
|
21
23
|
{
|
|
22
|
-
win: "Windows NT #{%w
|
|
23
|
-
linux: "X11; #{[
|
|
24
|
+
win: "Windows NT #{"#{%w[5 6 7 8 10].sample}.#{%w[0 1].sample}"}",
|
|
25
|
+
linux: "X11; #{['', "#{OS_DATA[:linux_distro].sample}; "].sample}Linux #{OS_DATA[:arch].sample}",
|
|
24
26
|
mac: "Macintosh; Intel Mac OS X 10_#{Random.rand(11..13)}_#{Random.rand(0..3)}"
|
|
25
27
|
}
|
|
26
28
|
end
|
|
27
29
|
|
|
28
30
|
def browser
|
|
29
|
-
chrome_version = "Chrome
|
|
30
|
-
|
|
31
|
-
["Gecko/20100101 Firefox/#{Random.rand(
|
|
31
|
+
chrome_version = "Chrome/#{Random.rand(90..92)}.0.#{Random.rand(1000..4000)}.#{Random.rand(100..900)}"
|
|
32
|
+
webkit603 = "603.#{Random.rand(0..9)}.#{Random.rand(0..9)}"
|
|
33
|
+
["Gecko/20100101 Firefox/#{Random.rand(82..90)}.0",
|
|
32
34
|
"AppleWebKit/537.36 (KHTML, like Gecko) #{chrome_version} Safari/537.36",
|
|
33
|
-
"AppleWebKit/#{
|
|
34
|
-
].sample
|
|
35
|
+
"AppleWebKit/#{webkit603} (KHTML, like Gecko) Version/10.1 #{chrome_version} Safari/#{webkit603}"].sample
|
|
35
36
|
end
|
|
36
37
|
end
|
|
37
38
|
end
|
data/lib/tweakphoeus/version.rb
CHANGED
data/lib/tweakphoeus.rb
CHANGED
|
@@ -1,87 +1,46 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'tweakphoeus/version'
|
|
4
|
+
require 'tweakphoeus/user_agent'
|
|
5
|
+
require 'tweakphoeus/cookie_jar'
|
|
6
|
+
require 'tweakphoeus/referer_list'
|
|
7
|
+
require 'typhoeus'
|
|
4
8
|
|
|
5
9
|
module Tweakphoeus
|
|
10
|
+
# Http client class
|
|
6
11
|
class Client
|
|
7
|
-
attr_accessor :cookie_jar
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def initialize(ua_systems: [
|
|
11
|
-
@cookie_jar =
|
|
12
|
-
@
|
|
13
|
-
@
|
|
14
|
-
|
|
15
|
-
"Accept-Language" => "es-ES,es;q=0.9,en;q=0.8",
|
|
16
|
-
"Accept-Encoding" => "",
|
|
17
|
-
"Connection" => "keep-alive"
|
|
18
|
-
}
|
|
12
|
+
attr_accessor :cookie_jar, :referer_list, :base_headers, :redirect
|
|
13
|
+
attr_reader :ssl_verifypeer
|
|
14
|
+
|
|
15
|
+
def initialize(ua_systems: %i[linux mac], ssl_verifypeer: true, redirect: false)
|
|
16
|
+
@cookie_jar = Tweakphoeus::CookieJar.new
|
|
17
|
+
@referer_list = Tweakphoeus::RefererList.new
|
|
18
|
+
@redirect = redirect
|
|
19
|
+
@base_headers = build_base_headers(ua_systems)
|
|
19
20
|
@proxy = nil
|
|
20
21
|
@proxyuserpwd = nil
|
|
21
|
-
@ssl_verifypeer =
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def get(url, body: nil, params: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
|
|
25
|
-
set_referer_from_headers(headers)
|
|
26
|
-
http_request(url, body: body, params: params, headers: headers, redirect: redirect, method: :get, ssl_verifypeer: ssl_verifypeer)
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def delete(url, body: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
|
|
30
|
-
set_referer_from_headers(headers)
|
|
31
|
-
http_request(url, body: body, headers: headers, redirect: redirect, method: :delete, ssl_verifypeer: ssl_verifypeer)
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def post(url, body: nil, params: nil, headers: nil, redirect: false, ssl_verifypeer: @ssl_verifypeer)
|
|
35
|
-
set_referer_from_headers(headers)
|
|
36
|
-
http_request(url, body: body, params: nil, headers: headers, redirect: redirect, method: :post, ssl_verifypeer: ssl_verifypeer)
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
def get_hide_inputs response
|
|
40
|
-
#TODO
|
|
22
|
+
@ssl_verifypeer = ssl_verifypeer
|
|
41
23
|
end
|
|
42
24
|
|
|
43
|
-
def
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
@cookie_jar[domain][key] = value
|
|
25
|
+
def get(url, body: nil, params: nil, headers: nil)
|
|
26
|
+
referer_from_headers(headers)
|
|
27
|
+
http_request(url, body: body, params: params, headers: headers, method: :get)
|
|
47
28
|
end
|
|
48
29
|
|
|
49
|
-
def
|
|
50
|
-
|
|
30
|
+
def delete(url, body: nil, headers: nil)
|
|
31
|
+
referer_from_headers(headers)
|
|
32
|
+
http_request(url, body: body, headers: headers, method: :delete)
|
|
51
33
|
end
|
|
52
34
|
|
|
53
|
-
def
|
|
54
|
-
|
|
55
|
-
url
|
|
35
|
+
def post(url, body: nil, headers: nil)
|
|
36
|
+
referer_from_headers(headers)
|
|
37
|
+
http_request(url, body: body, headers: headers, method: :post)
|
|
56
38
|
end
|
|
57
39
|
|
|
58
|
-
def
|
|
59
|
-
|
|
60
|
-
end
|
|
40
|
+
def set_proxy(url, auth)
|
|
41
|
+
return unless auth
|
|
61
42
|
|
|
62
|
-
|
|
63
|
-
@referer.last
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def cookie_string(url, headers={})
|
|
67
|
-
domain = get_domain(url)
|
|
68
|
-
headers ||= {}
|
|
69
|
-
cookies = parse_cookie(headers["Cookie"])
|
|
70
|
-
|
|
71
|
-
while domain != ""
|
|
72
|
-
if @cookie_jar[domain]
|
|
73
|
-
@cookie_jar[domain].each do |key, value|
|
|
74
|
-
cookies[key] ||= value
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
domain = domain.gsub(/^([^\.]+\.?)/, '')
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
cookies.map{ |key, value| "#{key}=#{value}" }.join('; ')
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def set_proxy(url, auth = nil)
|
|
84
|
-
@proxyuserpwd = "#{auth[:user]}:#{auth[:password]}" if auth.is_a?(Hash)
|
|
43
|
+
@proxyuserpwd = "#{auth[:user]}:#{auth[:password]}"
|
|
85
44
|
@proxy = url
|
|
86
45
|
end
|
|
87
46
|
|
|
@@ -92,94 +51,53 @@ module Tweakphoeus
|
|
|
92
51
|
|
|
93
52
|
private
|
|
94
53
|
|
|
95
|
-
def http_request(url, body: nil, params: nil, headers: nil,
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
request_headers["Referer"] = get_referer
|
|
99
|
-
response = Typhoeus.send(method, url, body: body, params: params, headers: request_headers, proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
|
|
100
|
-
obtain_cookies(response)
|
|
101
|
-
set_referer(url) if method != :post
|
|
102
|
-
if redirect && has_redirect?(response)
|
|
103
|
-
if response.code != 307
|
|
104
|
-
method = :get
|
|
105
|
-
body = nil
|
|
106
|
-
end
|
|
107
|
-
response = http_request(redirect_url(response),
|
|
108
|
-
body: body,
|
|
109
|
-
headers: headers,
|
|
110
|
-
redirect: redirect,
|
|
111
|
-
method: method,
|
|
112
|
-
ssl_verifypeer: ssl_verifypeer)
|
|
113
|
-
end
|
|
114
|
-
response
|
|
115
|
-
end
|
|
54
|
+
def http_request(url, body: nil, params: nil, headers: nil, method: :get)
|
|
55
|
+
response = Typhoeus.send(method, url, body: body, params: params, headers: build_request_headers(headers),
|
|
56
|
+
proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
|
|
116
57
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
end
|
|
58
|
+
@cookie_jar.obtain_cookies(response)
|
|
59
|
+
@referer_list.push_referer(url) if method != :post
|
|
120
60
|
|
|
121
|
-
|
|
122
|
-
set_cookies_field = response.headers["Set-Cookie"]
|
|
123
|
-
return if set_cookies_field.nil?
|
|
124
|
-
if set_cookies_field.is_a?(String)
|
|
125
|
-
set_cookies_field = [set_cookies_field]
|
|
126
|
-
end
|
|
61
|
+
return process_redirect(response) if redirect && redirect?(response)
|
|
127
62
|
|
|
128
|
-
|
|
129
|
-
key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..-1]
|
|
130
|
-
domain = cookie.match(/Domain=([^;]+)/)
|
|
131
|
-
|
|
132
|
-
if domain.nil?
|
|
133
|
-
domain = get_domain(response.request.url)
|
|
134
|
-
else
|
|
135
|
-
domain = domain[1].gsub(/^\./,'')
|
|
136
|
-
end
|
|
137
|
-
|
|
138
|
-
value = value.split(';').first
|
|
139
|
-
if value != "\"\""
|
|
140
|
-
@cookie_jar[domain] ||= {}
|
|
141
|
-
@cookie_jar[domain][key] = value
|
|
142
|
-
else
|
|
143
|
-
@cookie_jar[domain] ||= {}
|
|
144
|
-
@cookie_jar[domain].delete(key)
|
|
145
|
-
end
|
|
146
|
-
end
|
|
63
|
+
response
|
|
147
64
|
end
|
|
148
65
|
|
|
149
|
-
def
|
|
150
|
-
|
|
66
|
+
def build_request_headers(headers)
|
|
67
|
+
request_headers = merge_default_headers(headers)
|
|
68
|
+
request_headers['Cookie'] = @cookie_jar.cookie_string(url, headers)
|
|
69
|
+
request_headers['Referer'] = @referer_list.last_referer
|
|
70
|
+
end
|
|
151
71
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
cookies[key.strip] = value if value && !["Domain","Path","domain","path"].include?(key)
|
|
157
|
-
end
|
|
72
|
+
def process_redirect(response)
|
|
73
|
+
if response.code != 307
|
|
74
|
+
method = :get
|
|
75
|
+
body = nil
|
|
158
76
|
end
|
|
159
77
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
def has_redirect? response
|
|
164
|
-
!redirect_url(response).nil?
|
|
78
|
+
http_request(redirect_url(response), body: body, headers: headers,
|
|
79
|
+
redirect: redirect, method: method, ssl_verifypeer: ssl_verifypeer)
|
|
165
80
|
end
|
|
166
81
|
|
|
167
|
-
def
|
|
168
|
-
|
|
82
|
+
def build_base_headers(ua_systems)
|
|
83
|
+
{
|
|
84
|
+
'User-Agent' => UserAgent.random(systems: ua_systems),
|
|
85
|
+
'Accept-Language' => 'es-ES,es;q=0.9,en;q=0.8',
|
|
86
|
+
'Accept-Encoding' => '',
|
|
87
|
+
'Connection' => 'keep-alive'
|
|
88
|
+
}
|
|
169
89
|
end
|
|
170
90
|
|
|
171
|
-
def
|
|
172
|
-
|
|
91
|
+
def merge_default_headers(headers)
|
|
92
|
+
headers ? @base_headers.merge(headers) : @base_headers
|
|
173
93
|
end
|
|
174
94
|
|
|
175
|
-
def
|
|
176
|
-
|
|
177
|
-
@referer.last.replace headers["Referer"]
|
|
178
|
-
end
|
|
95
|
+
def redirect?(response)
|
|
96
|
+
!redirect_url(response).nil?
|
|
179
97
|
end
|
|
180
98
|
|
|
181
|
-
def
|
|
182
|
-
|
|
99
|
+
def redirect_url(response)
|
|
100
|
+
response.headers['Location']
|
|
183
101
|
end
|
|
184
102
|
end
|
|
185
103
|
end
|
data/tweakphoeus.gemspec
CHANGED
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
require 'tweakphoeus/version'
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'lib/tweakphoeus/version'
|
|
5
4
|
|
|
6
5
|
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name =
|
|
6
|
+
spec.name = 'tweakphoeus'
|
|
8
7
|
spec.version = Tweakphoeus::VERSION
|
|
9
|
-
spec.authors = [
|
|
10
|
-
spec.email = [
|
|
8
|
+
spec.authors = ['David Martin Garcia']
|
|
9
|
+
spec.email = ['davidmartingarcia@gmail.com']
|
|
11
10
|
|
|
12
|
-
spec.summary =
|
|
13
|
-
spec.description =
|
|
14
|
-
spec.homepage =
|
|
15
|
-
spec.license =
|
|
11
|
+
spec.summary = 'Typhoeus on steroids.'
|
|
12
|
+
spec.description = 'Typhoeus wrapper with some extras.'
|
|
13
|
+
spec.homepage = 'https://github.com/basestylo/Tweakphoeus/'
|
|
14
|
+
spec.license = 'MIT'
|
|
15
|
+
spec.required_ruby_version = '>= 2.6.0'
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
|
21
|
-
else
|
|
22
|
-
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
|
23
|
-
end
|
|
17
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
|
18
|
+
spec.metadata['source_code_uri'] = 'https://github.com/dmartingarcia/Tweakphoeus'
|
|
19
|
+
spec.metadata['changelog_uri'] = 'https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md'
|
|
24
20
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
spec.
|
|
28
|
-
|
|
21
|
+
# Specify which files should be added to the gem when it is released.
|
|
22
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
23
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
|
24
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
|
25
|
+
end
|
|
26
|
+
spec.bindir = 'exe'
|
|
27
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
|
28
|
+
spec.require_paths = ['lib']
|
|
29
29
|
|
|
30
|
-
spec.add_dependency 'typhoeus', '~> 1.
|
|
31
|
-
spec.add_development_dependency
|
|
32
|
-
spec.add_development_dependency
|
|
33
|
-
spec.add_development_dependency
|
|
30
|
+
spec.add_dependency 'typhoeus', '~> 1.4'
|
|
31
|
+
spec.add_development_dependency 'bundler', '~> 2.2'
|
|
32
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
|
33
|
+
spec.add_development_dependency 'rspec', '~> 3.10'
|
|
34
34
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: tweakphoeus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David Martin Garcia
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-04-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: typhoeus
|
|
@@ -16,78 +16,71 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1.
|
|
20
|
-
- - ">="
|
|
21
|
-
- !ruby/object:Gem::Version
|
|
22
|
-
version: 1.3.0
|
|
19
|
+
version: '1.4'
|
|
23
20
|
type: :runtime
|
|
24
21
|
prerelease: false
|
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
26
23
|
requirements:
|
|
27
24
|
- - "~>"
|
|
28
25
|
- !ruby/object:Gem::Version
|
|
29
|
-
version: '1.
|
|
30
|
-
- - ">="
|
|
31
|
-
- !ruby/object:Gem::Version
|
|
32
|
-
version: 1.3.0
|
|
26
|
+
version: '1.4'
|
|
33
27
|
- !ruby/object:Gem::Dependency
|
|
34
28
|
name: bundler
|
|
35
29
|
requirement: !ruby/object:Gem::Requirement
|
|
36
30
|
requirements:
|
|
37
31
|
- - "~>"
|
|
38
32
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
33
|
+
version: '2.2'
|
|
40
34
|
type: :development
|
|
41
35
|
prerelease: false
|
|
42
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
43
37
|
requirements:
|
|
44
38
|
- - "~>"
|
|
45
39
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '
|
|
40
|
+
version: '2.2'
|
|
47
41
|
- !ruby/object:Gem::Dependency
|
|
48
42
|
name: rake
|
|
49
43
|
requirement: !ruby/object:Gem::Requirement
|
|
50
44
|
requirements:
|
|
51
45
|
- - "~>"
|
|
52
46
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '
|
|
47
|
+
version: '13.0'
|
|
54
48
|
type: :development
|
|
55
49
|
prerelease: false
|
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
57
51
|
requirements:
|
|
58
52
|
- - "~>"
|
|
59
53
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '
|
|
54
|
+
version: '13.0'
|
|
61
55
|
- !ruby/object:Gem::Dependency
|
|
62
56
|
name: rspec
|
|
63
57
|
requirement: !ruby/object:Gem::Requirement
|
|
64
58
|
requirements:
|
|
65
59
|
- - "~>"
|
|
66
60
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '3.
|
|
68
|
-
- - ">"
|
|
69
|
-
- !ruby/object:Gem::Version
|
|
70
|
-
version: 3.0.0
|
|
61
|
+
version: '3.10'
|
|
71
62
|
type: :development
|
|
72
63
|
prerelease: false
|
|
73
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
74
65
|
requirements:
|
|
75
66
|
- - "~>"
|
|
76
67
|
- !ruby/object:Gem::Version
|
|
77
|
-
version: '3.
|
|
78
|
-
|
|
79
|
-
- !ruby/object:Gem::Version
|
|
80
|
-
version: 3.0.0
|
|
81
|
-
description: 'Typhoeus wrapper with new pretty things: cookie handler for example.'
|
|
68
|
+
version: '3.10'
|
|
69
|
+
description: Typhoeus wrapper with some extras.
|
|
82
70
|
email:
|
|
83
|
-
-
|
|
71
|
+
- davidmartingarcia@gmail.com
|
|
84
72
|
executables: []
|
|
85
73
|
extensions: []
|
|
86
74
|
extra_rdoc_files: []
|
|
87
75
|
files:
|
|
76
|
+
- ".github/workflows/auto-tag.yaml"
|
|
77
|
+
- ".github/workflows/main.yml"
|
|
78
|
+
- ".github/workflows/on-pull-request.yaml"
|
|
79
|
+
- ".github/workflows/on_new_tag.yaml"
|
|
80
|
+
- ".github/workflows/version-forget-me-not.yaml"
|
|
88
81
|
- ".gitignore"
|
|
89
82
|
- ".rspec"
|
|
90
|
-
- ".
|
|
83
|
+
- ".rubocop.yml"
|
|
91
84
|
- CHANGELOG.md
|
|
92
85
|
- CODE_OF_CONDUCT.md
|
|
93
86
|
- Gemfile
|
|
@@ -97,6 +90,9 @@ files:
|
|
|
97
90
|
- bin/console
|
|
98
91
|
- bin/setup
|
|
99
92
|
- lib/tweakphoeus.rb
|
|
93
|
+
- lib/tweakphoeus/cookie_jar.rb
|
|
94
|
+
- lib/tweakphoeus/helpers.rb
|
|
95
|
+
- lib/tweakphoeus/referer_list.rb
|
|
100
96
|
- lib/tweakphoeus/user_agent.rb
|
|
101
97
|
- lib/tweakphoeus/version.rb
|
|
102
98
|
- tweakphoeus.gemspec
|
|
@@ -104,7 +100,9 @@ homepage: https://github.com/basestylo/Tweakphoeus/
|
|
|
104
100
|
licenses:
|
|
105
101
|
- MIT
|
|
106
102
|
metadata:
|
|
107
|
-
|
|
103
|
+
homepage_uri: https://github.com/basestylo/Tweakphoeus/
|
|
104
|
+
source_code_uri: https://github.com/dmartingarcia/Tweakphoeus
|
|
105
|
+
changelog_uri: https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md
|
|
108
106
|
post_install_message:
|
|
109
107
|
rdoc_options: []
|
|
110
108
|
require_paths:
|
|
@@ -113,15 +111,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
113
111
|
requirements:
|
|
114
112
|
- - ">="
|
|
115
113
|
- !ruby/object:Gem::Version
|
|
116
|
-
version:
|
|
114
|
+
version: 2.6.0
|
|
117
115
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
116
|
requirements:
|
|
119
117
|
- - ">="
|
|
120
118
|
- !ruby/object:Gem::Version
|
|
121
119
|
version: '0'
|
|
122
120
|
requirements: []
|
|
123
|
-
|
|
124
|
-
rubygems_version: 2.7.3
|
|
121
|
+
rubygems_version: 3.1.2
|
|
125
122
|
signing_key:
|
|
126
123
|
specification_version: 4
|
|
127
124
|
summary: Typhoeus on steroids.
|
data/.travis.yml
DELETED