tweakphoeus 0.4.1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/auto-tag.yaml +15 -0
- data/.github/workflows/main.yml +21 -0
- data/.github/workflows/on_new_tag.yaml +17 -0
- data/.github/workflows/version-forget-me-not.yaml +17 -0
- data/CHANGELOG.md +19 -0
- data/README.md +1 -4
- data/lib/tweakphoeus.rb +15 -13
- data/lib/tweakphoeus/user_agent.rb +5 -5
- data/lib/tweakphoeus/version.rb +1 -1
- data/tweakphoeus.gemspec +21 -19
- metadata +22 -29
- data/.travis.yml +0 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 75e41d28b98ec62b97907e6adb7fa885a1fafc784045717fa44611e4aa172a68
|
4
|
+
data.tar.gz: b8c45de633c250c6c06a6307ce8cf023a8763c4579afc0513ac153e896eded6e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21c2be488160c13d8b5202f80508e6960d0f07348880f7170bee2d541cd433cca8e865b0c7fd9eb1342f2749e9d5fa540b1ad83938ad77e3bb6b236e6b70bedd
|
7
|
+
data.tar.gz: b1601abc1b9e426f2c3bc6e729464739ccebd5b3eae5f6f80c91982558a05f18352c395450be3cd50327b2f0278894f448bd91a0cf079a0305be5f1207860fbe
|
@@ -0,0 +1,21 @@
|
|
1
|
+
name: Ruby
|
2
|
+
|
3
|
+
on: [pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
steps:
|
9
|
+
- uses: actions/checkout@v2
|
10
|
+
- name: Set up Ruby
|
11
|
+
uses: ruby/setup-ruby@v1
|
12
|
+
with:
|
13
|
+
ruby-version: 2.7.4
|
14
|
+
bundler-cache: true
|
15
|
+
- name: Run the default task
|
16
|
+
run: bundle exec rake
|
17
|
+
services:
|
18
|
+
whoami:
|
19
|
+
image: containous/whoami
|
20
|
+
ports:
|
21
|
+
- 8080:80
|
@@ -0,0 +1,17 @@
|
|
1
|
+
name: On new tag
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
tags:
|
6
|
+
- '*'
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
publish-new-version:
|
10
|
+
runs-on: ubuntu-latest
|
11
|
+
steps:
|
12
|
+
- uses: actions/checkout@v2
|
13
|
+
- name: Publish gem
|
14
|
+
uses: dawidd6/action-publish-gem@v1
|
15
|
+
with:
|
16
|
+
api_key: ${{secrets.RUBYGEMS_API_KEY}}
|
17
|
+
github_token: ${{secrets.GITHUB_TOKEN}}
|
@@ -0,0 +1,17 @@
|
|
1
|
+
name: Version Forget-Me-Not
|
2
|
+
|
3
|
+
on:
|
4
|
+
pull_request:
|
5
|
+
branches:
|
6
|
+
- master
|
7
|
+
- main
|
8
|
+
types: [opened, synchronize]
|
9
|
+
|
10
|
+
jobs:
|
11
|
+
semver-check:
|
12
|
+
runs-on: ubuntu-20.04
|
13
|
+
steps:
|
14
|
+
- uses: simplybusiness/version-forget-me-not@v2.1.0
|
15
|
+
env:
|
16
|
+
ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
17
|
+
VERSION_FILE_PATH: "lib/tweakphoeus/version.rb"
|
data/CHANGELOG.md
CHANGED
@@ -15,3 +15,22 @@
|
|
15
15
|
-- v0.3.0 - adding HTTP proxy management and fixing circular reference
|
16
16
|
|
17
17
|
-- v0.4.1 - adding random User-Agent generator
|
18
|
+
|
19
|
+
-- v0.4.2 - adding ssl verification & deletion of active support methods
|
20
|
+
|
21
|
+
-- v0.4.5 - pushing typhoeus version
|
22
|
+
|
23
|
+
-- v0.5.0 - Upgrading dependencies and revamping development environment
|
24
|
+
|
25
|
+
Bumping required ruby version to >=2.6.0
|
26
|
+
|
27
|
+
|
28
|
+
Upgrading Typhoeus dependency to 1.4.0
|
29
|
+
|
30
|
+
|
31
|
+
Upgrading Development dependencies:
|
32
|
+
- Bundler 2.2
|
33
|
+
- Rake 13.0
|
34
|
+
- Rspec 3.10
|
35
|
+
|
36
|
+
Swapping CI environment from CircleCI to Github Actions
|
data/README.md
CHANGED
@@ -1,13 +1,11 @@
|
|
1
|
-
[![Stories in Ready](https://badge.waffle.io/basestylo/Tweakphoeus.png?label=ready&title=Ready)](https://waffle.io/basestylo/Tweakphoeus)
|
2
1
|
[![Gem Version](https://badge.fury.io/rb/tweakphoeus.svg)](https://badge.fury.io/rb/tweakphoeus)
|
3
|
-
[![Circle CI](https://circleci.com/gh/basestylo/Tweakphoeus/tree/master.svg?style=svg)](https://circleci.com/gh/basestylo/Tweakphoeus/tree/master)
|
4
2
|
[![Code Climate](https://codeclimate.com/github/basestylo/Tweakphoeus/badges/gpa.svg)](https://codeclimate.com/github/basestylo/Tweakphoeus)
|
5
3
|
[![Issue Count](https://codeclimate.com/github/basestylo/Tweakphoeus/badges/issue_count.svg)](https://codeclimate.com/github/basestylo/Tweakphoeus)
|
6
4
|
# Tweakphoeus
|
7
5
|
|
8
6
|
We usually describe this gem as 'typhoeus on steroids'. We add some browser features, for example cookies management and automation in tipical headers that browsers define in his HTTP stack.
|
9
7
|
|
10
|
-
We love scrapping and this gem was created for this.
|
8
|
+
We love scrapping and this gem was created for this.
|
11
9
|
|
12
10
|
From crazy developers to another crazy developers, Created in Bizneo.com and maintained in this small group of developers.
|
13
11
|
|
@@ -53,4 +51,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
|
|
53
51
|
## License
|
54
52
|
|
55
53
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
56
|
-
|
data/lib/tweakphoeus.rb
CHANGED
@@ -7,32 +7,33 @@ module Tweakphoeus
|
|
7
7
|
attr_accessor :cookie_jar
|
8
8
|
attr_accessor :base_headers
|
9
9
|
|
10
|
-
def initialize()
|
10
|
+
def initialize(ua_systems: [:linux, :mac])
|
11
11
|
@cookie_jar = {}
|
12
12
|
@referer = [""]
|
13
13
|
@base_headers = {
|
14
|
-
"User-Agent" => UserAgent.random(systems:
|
15
|
-
"Accept-Language" => "es-ES,es;q=0.
|
14
|
+
"User-Agent" => UserAgent.random(systems: ua_systems),
|
15
|
+
"Accept-Language" => "es-ES,es;q=0.9,en;q=0.8",
|
16
16
|
"Accept-Encoding" => "",
|
17
17
|
"Connection" => "keep-alive"
|
18
18
|
}
|
19
19
|
@proxy = nil
|
20
20
|
@proxyuserpwd = nil
|
21
|
+
@ssl_verifypeer = true
|
21
22
|
end
|
22
23
|
|
23
|
-
def get(url, body: nil, params: nil, headers: nil, redirect: true)
|
24
|
+
def get(url, body: nil, params: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
|
24
25
|
set_referer_from_headers(headers)
|
25
|
-
http_request(url, body: body, params: params, headers: headers, redirect: redirect, method: :get)
|
26
|
+
http_request(url, body: body, params: params, headers: headers, redirect: redirect, method: :get, ssl_verifypeer: ssl_verifypeer)
|
26
27
|
end
|
27
28
|
|
28
|
-
def delete(url, body: nil, headers: nil, redirect: true)
|
29
|
+
def delete(url, body: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
|
29
30
|
set_referer_from_headers(headers)
|
30
|
-
http_request(url, body: body, headers: headers, redirect: redirect, method: :delete)
|
31
|
+
http_request(url, body: body, headers: headers, redirect: redirect, method: :delete, ssl_verifypeer: ssl_verifypeer)
|
31
32
|
end
|
32
33
|
|
33
|
-
def post(url, body: nil, params: nil, headers: nil, redirect: false)
|
34
|
+
def post(url, body: nil, params: nil, headers: nil, redirect: false, ssl_verifypeer: @ssl_verifypeer)
|
34
35
|
set_referer_from_headers(headers)
|
35
|
-
http_request(url, body: body, params: nil, headers: headers, redirect: redirect, method: :post)
|
36
|
+
http_request(url, body: body, params: nil, headers: headers, redirect: redirect, method: :post, ssl_verifypeer: ssl_verifypeer)
|
36
37
|
end
|
37
38
|
|
38
39
|
def get_hide_inputs response
|
@@ -67,7 +68,7 @@ module Tweakphoeus
|
|
67
68
|
headers ||= {}
|
68
69
|
cookies = parse_cookie(headers["Cookie"])
|
69
70
|
|
70
|
-
while domain
|
71
|
+
while domain != ""
|
71
72
|
if @cookie_jar[domain]
|
72
73
|
@cookie_jar[domain].each do |key, value|
|
73
74
|
cookies[key] ||= value
|
@@ -91,11 +92,11 @@ module Tweakphoeus
|
|
91
92
|
|
92
93
|
private
|
93
94
|
|
94
|
-
def http_request(url, body: nil, params: nil, headers: nil, redirect: false, method: :get)
|
95
|
+
def http_request(url, body: nil, params: nil, headers: nil, redirect: false, method: :get, ssl_verifypeer: @ssl_verifypeer)
|
95
96
|
request_headers = merge_default_headers(headers)
|
96
97
|
request_headers["Cookie"] = cookie_string(url, headers)
|
97
98
|
request_headers["Referer"] = get_referer
|
98
|
-
response = Typhoeus.send(method, url, body: body, params: params, headers: request_headers, proxy: @proxy, proxyuserpwd: @proxyuserpwd)
|
99
|
+
response = Typhoeus.send(method, url, body: body, params: params, headers: request_headers, proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
|
99
100
|
obtain_cookies(response)
|
100
101
|
set_referer(url) if method != :post
|
101
102
|
if redirect && has_redirect?(response)
|
@@ -107,7 +108,8 @@ module Tweakphoeus
|
|
107
108
|
body: body,
|
108
109
|
headers: headers,
|
109
110
|
redirect: redirect,
|
110
|
-
method: method
|
111
|
+
method: method,
|
112
|
+
ssl_verifypeer: ssl_verifypeer)
|
111
113
|
end
|
112
114
|
response
|
113
115
|
end
|
@@ -10,8 +10,8 @@ module Tweakphoeus
|
|
10
10
|
class << self
|
11
11
|
def random(systems: OS_DATA[:types])
|
12
12
|
user_agent = "Mozilla/5.0 (#{os[systems.sample]}) #{browser}"
|
13
|
-
firefox_version = user_agent.match(/Firefox\/([0-9\.]+)/)
|
14
|
-
user_agent.gsub!(')', "; rv:#{firefox_version})") if firefox_version
|
13
|
+
firefox_version = user_agent.match(/Firefox\/([0-9\.]+)/)
|
14
|
+
user_agent.gsub!(')', "; rv:#{firefox_version[1]})") if firefox_version.is_a?(MatchData)
|
15
15
|
user_agent
|
16
16
|
end
|
17
17
|
|
@@ -21,14 +21,14 @@ module Tweakphoeus
|
|
21
21
|
{
|
22
22
|
win: "Windows NT #{%w(5 6 7 8 10).sample + '.' + %w(0 1).sample}",
|
23
23
|
linux: "X11; #{["", OS_DATA[:linux_distro].sample+"; "].sample}Linux #{OS_DATA[:arch].sample}",
|
24
|
-
mac: "Macintosh; Intel Mac OS X 10_#{Random.rand(
|
24
|
+
mac: "Macintosh; Intel Mac OS X 10_#{Random.rand(11..13)}_#{Random.rand(0..3)}"
|
25
25
|
}
|
26
26
|
end
|
27
27
|
|
28
28
|
def browser
|
29
|
-
chrome_version = "Chrome/
|
29
|
+
chrome_version = "Chrome/6#{Random.rand(0..5)}.0.#{Random.rand(1000..4000)}.#{Random.rand(100..900)}"
|
30
30
|
webkit_603 = "603.#{Random.rand(0..9)}.#{Random.rand(0..9)}"
|
31
|
-
["Gecko/20100101 Firefox/#{Random.rand(
|
31
|
+
["Gecko/20100101 Firefox/#{Random.rand(55..59)}.0",
|
32
32
|
"AppleWebKit/537.36 (KHTML, like Gecko) #{chrome_version} Safari/537.36",
|
33
33
|
"AppleWebKit/#{webkit_603} (KHTML, like Gecko) Version/10.1 #{chrome_version} Safari/#{webkit_603}"
|
34
34
|
].sample
|
data/lib/tweakphoeus/version.rb
CHANGED
data/tweakphoeus.gemspec
CHANGED
@@ -1,34 +1,36 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
|
4
|
-
require 'tweakphoeus/version'
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/tweakphoeus/version"
|
5
4
|
|
6
5
|
Gem::Specification.new do |spec|
|
7
6
|
spec.name = "tweakphoeus"
|
8
7
|
spec.version = Tweakphoeus::VERSION
|
9
8
|
spec.authors = ["David Martin Garcia"]
|
10
|
-
spec.email = ["
|
9
|
+
spec.email = ["davidmartingarcia@gmail.com"]
|
11
10
|
|
12
11
|
spec.summary = %q{Typhoeus on steroids.}
|
13
|
-
spec.description = %q{Typhoeus wrapper with
|
12
|
+
spec.description = %q{Typhoeus wrapper with some extras.}
|
14
13
|
spec.homepage = "https://github.com/basestylo/Tweakphoeus/"
|
15
14
|
spec.license = "MIT"
|
15
|
+
spec.required_ruby_version = ">= 2.6.0"
|
16
16
|
|
17
|
-
|
18
|
-
# delete this section to allow pushing this gem to any host.
|
19
|
-
if spec.respond_to?(:metadata)
|
20
|
-
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
21
|
-
else
|
22
|
-
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
|
-
end
|
17
|
+
spec.metadata["allowed_push_host"] = "https://rubygems.org/"
|
24
18
|
|
25
|
-
spec.
|
19
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
20
|
+
spec.metadata["source_code_uri"] = "https://github.com/dmartingarcia/Tweakphoeus"
|
21
|
+
spec.metadata["changelog_uri"] = "https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md"
|
22
|
+
|
23
|
+
# Specify which files should be added to the gem when it is released.
|
24
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
25
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
26
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
27
|
+
end
|
26
28
|
spec.bindir = "exe"
|
27
|
-
spec.executables = spec.files.grep(%r{
|
29
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
28
30
|
spec.require_paths = ["lib"]
|
29
31
|
|
30
|
-
spec.add_dependency 'typhoeus', '~> 1.
|
31
|
-
spec.add_development_dependency "bundler", "~>
|
32
|
-
spec.add_development_dependency "rake", "~>
|
33
|
-
spec.add_development_dependency "rspec", "~> 3.
|
32
|
+
spec.add_dependency 'typhoeus', '~> 1.4'
|
33
|
+
spec.add_development_dependency "bundler", "~> 2.2"
|
34
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
35
|
+
spec.add_development_dependency "rspec", "~> 3.10"
|
34
36
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tweakphoeus
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Martin Garcia
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-08-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: typhoeus
|
@@ -16,78 +16,69 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
20
|
-
- - ">="
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 1.0.1
|
19
|
+
version: '1.4'
|
23
20
|
type: :runtime
|
24
21
|
prerelease: false
|
25
22
|
version_requirements: !ruby/object:Gem::Requirement
|
26
23
|
requirements:
|
27
24
|
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
version: '1.
|
30
|
-
- - ">="
|
31
|
-
- !ruby/object:Gem::Version
|
32
|
-
version: 1.0.1
|
26
|
+
version: '1.4'
|
33
27
|
- !ruby/object:Gem::Dependency
|
34
28
|
name: bundler
|
35
29
|
requirement: !ruby/object:Gem::Requirement
|
36
30
|
requirements:
|
37
31
|
- - "~>"
|
38
32
|
- !ruby/object:Gem::Version
|
39
|
-
version: '
|
33
|
+
version: '2.2'
|
40
34
|
type: :development
|
41
35
|
prerelease: false
|
42
36
|
version_requirements: !ruby/object:Gem::Requirement
|
43
37
|
requirements:
|
44
38
|
- - "~>"
|
45
39
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
40
|
+
version: '2.2'
|
47
41
|
- !ruby/object:Gem::Dependency
|
48
42
|
name: rake
|
49
43
|
requirement: !ruby/object:Gem::Requirement
|
50
44
|
requirements:
|
51
45
|
- - "~>"
|
52
46
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
47
|
+
version: '13.0'
|
54
48
|
type: :development
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
51
|
requirements:
|
58
52
|
- - "~>"
|
59
53
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
54
|
+
version: '13.0'
|
61
55
|
- !ruby/object:Gem::Dependency
|
62
56
|
name: rspec
|
63
57
|
requirement: !ruby/object:Gem::Requirement
|
64
58
|
requirements:
|
65
59
|
- - "~>"
|
66
60
|
- !ruby/object:Gem::Version
|
67
|
-
version: '3.
|
68
|
-
- - ">"
|
69
|
-
- !ruby/object:Gem::Version
|
70
|
-
version: 3.0.0
|
61
|
+
version: '3.10'
|
71
62
|
type: :development
|
72
63
|
prerelease: false
|
73
64
|
version_requirements: !ruby/object:Gem::Requirement
|
74
65
|
requirements:
|
75
66
|
- - "~>"
|
76
67
|
- !ruby/object:Gem::Version
|
77
|
-
version: '3.
|
78
|
-
|
79
|
-
- !ruby/object:Gem::Version
|
80
|
-
version: 3.0.0
|
81
|
-
description: 'Typhoeus wrapper with new pretty things: cookie handler for example.'
|
68
|
+
version: '3.10'
|
69
|
+
description: Typhoeus wrapper with some extras.
|
82
70
|
email:
|
83
|
-
-
|
71
|
+
- davidmartingarcia@gmail.com
|
84
72
|
executables: []
|
85
73
|
extensions: []
|
86
74
|
extra_rdoc_files: []
|
87
75
|
files:
|
76
|
+
- ".github/workflows/auto-tag.yaml"
|
77
|
+
- ".github/workflows/main.yml"
|
78
|
+
- ".github/workflows/on_new_tag.yaml"
|
79
|
+
- ".github/workflows/version-forget-me-not.yaml"
|
88
80
|
- ".gitignore"
|
89
81
|
- ".rspec"
|
90
|
-
- ".travis.yml"
|
91
82
|
- CHANGELOG.md
|
92
83
|
- CODE_OF_CONDUCT.md
|
93
84
|
- Gemfile
|
@@ -104,7 +95,10 @@ homepage: https://github.com/basestylo/Tweakphoeus/
|
|
104
95
|
licenses:
|
105
96
|
- MIT
|
106
97
|
metadata:
|
107
|
-
allowed_push_host: https://rubygems.org
|
98
|
+
allowed_push_host: https://rubygems.org/
|
99
|
+
homepage_uri: https://github.com/basestylo/Tweakphoeus/
|
100
|
+
source_code_uri: https://github.com/dmartingarcia/Tweakphoeus
|
101
|
+
changelog_uri: https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md
|
108
102
|
post_install_message:
|
109
103
|
rdoc_options: []
|
110
104
|
require_paths:
|
@@ -113,15 +107,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
113
107
|
requirements:
|
114
108
|
- - ">="
|
115
109
|
- !ruby/object:Gem::Version
|
116
|
-
version:
|
110
|
+
version: 2.6.0
|
117
111
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
118
112
|
requirements:
|
119
113
|
- - ">="
|
120
114
|
- !ruby/object:Gem::Version
|
121
115
|
version: '0'
|
122
116
|
requirements: []
|
123
|
-
|
124
|
-
rubygems_version: 2.6.11
|
117
|
+
rubygems_version: 3.1.2
|
125
118
|
signing_key:
|
126
119
|
specification_version: 4
|
127
120
|
summary: Typhoeus on steroids.
|
data/.travis.yml
DELETED