tweakphoeus 0.4.5 → 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9f0ef33e29e152d356f04c714a256ee5676bec7071ba1f5385829b095c848804
4
- data.tar.gz: 9f83f26691034aa63ec9a58c7c1c642daa8b561012ad7a6b41eb8251496a8173
3
+ metadata.gz: c2fc171c1f446511f0c9ad337b4885e9ffc8cd0fe28ea3089875cf5a15a7fb25
4
+ data.tar.gz: 59583e9143549eb48d50b58e01e84155082e71c573191a3e4ce39f791ddd5e32
5
5
  SHA512:
6
- metadata.gz: 28963df49656e81acab37fc40dd5107a9564758413061500bf33d2b4e6b391ee4dbb5cbdd76ef2022eb945461f4897654ccb4a2f604efe141ac9f4542a29a560
7
- data.tar.gz: 3c071e4bd4c542c654ac5582c66bdbfdea4b3635f1b7a17ddb1c5af2c930b85159f8237358a35d8ae2ea0e95ef61381a23282f27c2a0b85888201280dc20dbde
6
+ metadata.gz: 746e43e153431a56c07a7ec5c51ffa1a90f4e800c1018cb3792214877e10edc2e0aa3278a6290a43231192da9045170615bf6f32b1e0d521bc7ccb25d5e8bd28
7
+ data.tar.gz: '043856a637bbfb0a9fb5cdfd27f1bcba6ea8e132f3fdff9e0ce119f32578d042b039668f855383570c0b7ff02553c6b087b2506a632fd56bfa4fcd8500e2fa96'
@@ -0,0 +1,15 @@
1
+ name: Auto-tag
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - master
7
+
8
+ jobs:
9
+ auto-tag:
10
+ runs-on: ubuntu-20.04
11
+ steps:
12
+ - uses: duderman/gh-gem-tag-action@v1
13
+ with:
14
+ github_token: ${{ secrets.GITHUB_TOKEN }}
15
+ tag_prefix: v
@@ -0,0 +1,24 @@
1
+ name: Ruby
2
+
3
+ on: [pull_request]
4
+
5
+ jobs:
6
+ build:
7
+ runs-on: ubuntu-latest
8
+ strategy:
9
+ matrix:
10
+ ruby: [ '2.6', '2.7', '3.0' ]
11
+ steps:
12
+ - uses: actions/checkout@v2
13
+ - name: Set up Ruby ${{ matrix.ruby }}
14
+ uses: ruby/setup-ruby@v1
15
+ with:
16
+ ruby-version: ${{ matrix.ruby }}
17
+ bundler-cache: true
18
+ - name: Run the default task
19
+ run: bundle exec rake
20
+ services:
21
+ whoami:
22
+ image: containous/whoami
23
+ ports:
24
+ - 8080:80
@@ -0,0 +1,23 @@
1
+ name: Github on-pull-request Action
2
+
3
+ on: [pull_request]
4
+
5
+ jobs:
6
+ rubocop:
7
+ runs-on: ubuntu-latest
8
+ steps:
9
+ - uses: actions/checkout@v2
10
+ - name: RuboCop Linter Action
11
+ uses: andrewmcodes/rubocop-linter-action@v3.3.0
12
+ env:
13
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
14
+ reek:
15
+ name: Reek
16
+ runs-on: ubuntu-latest
17
+ steps:
18
+ - uses: actions/checkout@v2
19
+ - uses: actions/setup-ruby@v1
20
+ with:
21
+ ruby-version: '2.7'
22
+ - run: gem install reek --no-doc
23
+ - run: reek .
@@ -0,0 +1,17 @@
1
+ name: On new tag
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - '*'
7
+
8
+ jobs:
9
+ publish-new-version:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v2
13
+ - name: Publish gem
14
+ uses: dawidd6/action-publish-gem@v1
15
+ with:
16
+ api_key: ${{secrets.RUBYGEMS_API_KEY}}
17
+ github_token: ${{secrets.GITHUB_TOKEN}}
@@ -0,0 +1,17 @@
1
+ name: Version Forget-Me-Not
2
+
3
+ on:
4
+ pull_request:
5
+ branches:
6
+ - master
7
+ - main
8
+ types: [opened, synchronize]
9
+
10
+ jobs:
11
+ semver-check:
12
+ runs-on: ubuntu-20.04
13
+ steps:
14
+ - uses: simplybusiness/version-forget-me-not@v2.1.0
15
+ env:
16
+ ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
17
+ VERSION_FILE_PATH: "lib/tweakphoeus/version.rb"
data/.rubocop.yml ADDED
@@ -0,0 +1,6 @@
1
+ AllCops:
2
+ NewCops: enable
3
+ TargetRubyVersion: 2.6
4
+
5
+ Layout/LineLength:
6
+ Max: 120
data/CHANGELOG.md CHANGED
@@ -19,3 +19,29 @@
19
19
  -- v0.4.2 - adding ssl verification & deletion of active support methods
20
20
 
21
21
  -- v0.4.5 - pushing typhoeus version
22
+
23
+ -- v0.5.0 - Upgrading dependencies and revamping development environment
24
+
25
+ Bumping required ruby version to >=2.6.0
26
+
27
+ Upgrading Development dependencies:
28
+ - Bundler 2.2
29
+ - Rake 13.0
30
+ - Rspec 3.10
31
+
32
+ Swapping CI environment from CircleCI to Github Actions
33
+
34
+ Upgrading Typhoeus dependency to 1.4.0
35
+
36
+ -- v0.6.0 - Major refactor of Tweakphoeus::Client API
37
+
38
+ - New interface for Tweakphoeus::Client
39
+ - Initializer now accepts ssl_verifypeer and redirect parameters instead of having it on all HTTP verb methods (get, delete and post)
40
+ - ssl_verifypeer and redirect parameters removed from HTTP verb methods
41
+ - cookie_jar is not longer an Hash, it has been extracted to Tweakphoeus::CookieJar class
42
+ - refeer_list is not longer an Array, it has been extracted to Tweakphoeus::RefererList class
43
+
44
+ -- v0.6.1 - Fixing issues with the modification of a frozen string
45
+ - Adds missing requires in the mail `.rb` file
46
+ - Solves an issue with a modification on a frozen string
47
+ > /Users/davidmartingarcia/.rbenv/versions/2.7.4/lib/ruby/gems/2.7.0/gems/tweakphoeus-0.6.0/lib/tweakphoeus/user_agent.rb:17:in `gsub!': can't modify frozen String: "Mozilla/5.0 (X11; gNewSense; Linux x86) Gecko/20100101 Firefox/84.0" (FrozenError)
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source 'https://rubygems.org'
2
4
 
3
5
  # Specify your gem's dependencies in tweakphoeus.gemspec
data/README.md CHANGED
@@ -1,13 +1,13 @@
1
- [![Stories in Ready](https://badge.waffle.io/basestylo/Tweakphoeus.png?label=ready&title=Ready)](https://waffle.io/basestylo/Tweakphoeus)
2
- [![Gem Version](https://badge.fury.io/rb/tweakphoeus.svg)](https://badge.fury.io/rb/tweakphoeus)
3
- [![Circle CI](https://circleci.com/gh/basestylo/Tweakphoeus/tree/master.svg?style=svg)](https://circleci.com/gh/basestylo/Tweakphoeus/tree/master)
4
- [![Code Climate](https://codeclimate.com/github/basestylo/Tweakphoeus/badges/gpa.svg)](https://codeclimate.com/github/basestylo/Tweakphoeus)
1
+ [![Gem Version](https://img.shields.io/gem/v/tweakphoeus.svg)](https://rubygems.org/gems/tweakphoeus)
2
+ [![Maintainability](https://api.codeclimate.com/v1/badges/f4c831c3308b1f53bc2a/maintainability)](https://codeclimate.com/github/basestylo/Tweakphoeus/maintainability)
3
+ [![GitHub license](https://img.shields.io/github/license/Naereen/StrapDown.js.svg)](https://github.com/Naereen/StrapDown.js/blob/master/LICENSE)
5
4
  [![Issue Count](https://codeclimate.com/github/basestylo/Tweakphoeus/badges/issue_count.svg)](https://codeclimate.com/github/basestylo/Tweakphoeus)
5
+
6
6
  # Tweakphoeus
7
7
 
8
8
  We usually describe this gem as 'typhoeus on steroids'. We add some browser features, for example cookies management and automation in tipical headers that browsers define in his HTTP stack.
9
9
 
10
- We love scrapping and this gem was created for this.
10
+ We love scrapping and this gem was created for this.
11
11
 
12
12
  From crazy developers to another crazy developers, Created in Bizneo.com and maintained in this small group of developers.
13
13
 
@@ -53,4 +53,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
53
53
  ## License
54
54
 
55
55
  The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
56
-
data/Rakefile CHANGED
@@ -1,6 +1,8 @@
1
- require "bundler/gem_tasks"
2
- require "rspec/core/rake_task"
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/gem_tasks'
4
+ require 'rspec/core/rake_task'
3
5
 
4
6
  RSpec::Core::RakeTask.new(:spec)
5
7
 
6
- task :default => :spec
8
+ task default: :spec
data/bin/console CHANGED
@@ -1,7 +1,8 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
 
3
- require "bundler/setup"
4
- require "tweakphoeus"
4
+ require 'bundler/setup'
5
+ require 'tweakphoeus'
5
6
 
6
7
  # You can add fixtures and/or initialization code here to make experimenting
7
8
  # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +11,5 @@ require "tweakphoeus"
10
11
  # require "pry"
11
12
  # Pry.start
12
13
 
13
- require "irb"
14
+ require 'irb'
14
15
  IRB.start
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'tweakphoeus/helpers'
4
+
5
+ module Tweakphoeus
6
+ # Cookie manager
7
+ class CookieJar
8
+ include Tweakphoeus::Helpers
9
+
10
+ def initialize
11
+ @cookie_jar = {}
12
+ end
13
+
14
+ def purge_bad_cookies(cookies)
15
+ cookies.reject { |cookie| cookie.first.last == '""' }
16
+ end
17
+
18
+ def cookie_string(url, headers = {})
19
+ domain = get_domain(url)
20
+ headers ||= {}
21
+ cookies = parse_cookie(headers['Cookie'])
22
+
23
+ while domain != ''
24
+ @cookie_jar[domain]&.each do |key, value|
25
+ cookies[key] ||= value
26
+ end
27
+ domain = domain.gsub(/^([^.]+\.?)/, '')
28
+ end
29
+
30
+ cookies.map { |key, value| "#{key}=#{value}" }.join('; ')
31
+ end
32
+
33
+ def obtain_cookies(response)
34
+ set_cookies_field = [response.headers['Set-Cookie']].compact
35
+
36
+ set_cookies_field.each do |cookie|
37
+ key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..]
38
+ domain = cookie.match(/Domain=([^;]+)/)&.at(1)&.gsub(/^\./, '')
39
+
40
+ domain = get_domain(response.request.url) if domain.nil
41
+ set_cookie(domain, key, value)
42
+ end
43
+ end
44
+
45
+ def add_cookies(host, key, value)
46
+ domain = get_domain(host)
47
+ @cookie_jar[domain] ||= {}
48
+ @cookie_jar[domain][key] = value
49
+ end
50
+
51
+ private
52
+
53
+ def set_cookie(domain, key, value)
54
+ @cookie_jar[domain] ||= {}
55
+
56
+ value = value.split(';').first
57
+
58
+ if value == '""'
59
+ @cookie_jar[domain].delete(key)
60
+ else
61
+ @cookie_jar[domain][key] = value
62
+ end
63
+ end
64
+
65
+ def parse_cookie(string)
66
+ cookies = {}
67
+
68
+ if string.is_a?(String)
69
+ string.split(';').each do |part|
70
+ key, value = part.split('=')
71
+ key.strip!
72
+ cookies[key.strip] = value if value && !%w[Domain Path domain path].include?(key)
73
+ end
74
+ end
75
+
76
+ cookies
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tweakphoeus
4
+ # Helper functions for common behavious across all classes
5
+ module Helpers
6
+ def get_domain(domain)
7
+ domain.match(%r{([a-zA-Z0-9]+://|)([^/]+)})[2].gsub(/^\./, '')
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tweakphoeus
4
+ # RefererList class handle referer logic for Referer header in requests
5
+ class RefererList
6
+ def initialize
7
+ @referer = []
8
+ end
9
+
10
+ def push_referer(url)
11
+ @referer << url
12
+ end
13
+
14
+ def pop_referer
15
+ @referer.pop || ''
16
+ end
17
+
18
+ def last_referer
19
+ @referer.last
20
+ end
21
+
22
+ def referer_from_headers(headers)
23
+ @referer.last.replace(headers['Referer']) if headers && headers['Referer'].is_a?(String)
24
+ end
25
+ end
26
+ end
@@ -1,37 +1,38 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Tweakphoeus
4
+ # User agent data generator
2
5
  class UserAgent
3
6
  OS_DATA = {
4
- types: [:win, :linux, :mac],
5
- linux_distro: ["Ubuntu", "Debian", "Fedora", "gNewSense", "Linux Mint", "OpenSUSE", "Mandriva"],
6
- arch: ["x86", "x86_64"],
7
- win_arch: ["Win64; x64", "WOW64"]
8
- }
7
+ types: %i[win linux mac],
8
+ linux_distro: ['Ubuntu', 'Debian', 'Fedora', 'gNewSense', 'Linux Mint', 'OpenSUSE', 'Mandriva'],
9
+ arch: %w[x86 x86_64],
10
+ win_arch: ['Win64; x64', 'WOW64']
11
+ }.freeze
9
12
 
10
13
  class << self
11
14
  def random(systems: OS_DATA[:types])
12
15
  user_agent = "Mozilla/5.0 (#{os[systems.sample]}) #{browser}"
13
- firefox_version = user_agent.match(/Firefox\/([0-9\.]+)/)
14
- user_agent.gsub!(')', "; rv:#{firefox_version[1]})") if firefox_version.is_a?(MatchData)
15
- user_agent
16
+ firefox_version = user_agent.match(%r{Firefox/([0-9.]+)})
17
+ user_agent.gsub(')', "; rv:#{firefox_version[1]})") if firefox_version.is_a?(MatchData)
16
18
  end
17
19
 
18
20
  private
19
21
 
20
22
  def os
21
23
  {
22
- win: "Windows NT #{%w(5 6 7 8 10).sample + '.' + %w(0 1).sample}",
23
- linux: "X11; #{["", OS_DATA[:linux_distro].sample+"; "].sample}Linux #{OS_DATA[:arch].sample}",
24
+ win: "Windows NT #{"#{%w[5 6 7 8 10].sample}.#{%w[0 1].sample}"}",
25
+ linux: "X11; #{['', "#{OS_DATA[:linux_distro].sample}; "].sample}Linux #{OS_DATA[:arch].sample}",
24
26
  mac: "Macintosh; Intel Mac OS X 10_#{Random.rand(11..13)}_#{Random.rand(0..3)}"
25
27
  }
26
28
  end
27
29
 
28
30
  def browser
29
- chrome_version = "Chrome/6#{Random.rand(0..5)}.0.#{Random.rand(1000..4000)}.#{Random.rand(100..900)}"
30
- webkit_603 = "603.#{Random.rand(0..9)}.#{Random.rand(0..9)}"
31
- ["Gecko/20100101 Firefox/#{Random.rand(55..59)}.0",
31
+ chrome_version = "Chrome/#{Random.rand(90..92)}.0.#{Random.rand(1000..4000)}.#{Random.rand(100..900)}"
32
+ webkit603 = "603.#{Random.rand(0..9)}.#{Random.rand(0..9)}"
33
+ ["Gecko/20100101 Firefox/#{Random.rand(82..90)}.0",
32
34
  "AppleWebKit/537.36 (KHTML, like Gecko) #{chrome_version} Safari/537.36",
33
- "AppleWebKit/#{webkit_603} (KHTML, like Gecko) Version/10.1 #{chrome_version} Safari/#{webkit_603}"
34
- ].sample
35
+ "AppleWebKit/#{webkit603} (KHTML, like Gecko) Version/10.1 #{chrome_version} Safari/#{webkit603}"].sample
35
36
  end
36
37
  end
37
38
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Tweakphoeus
2
- VERSION = "0.4.5"
4
+ VERSION = '0.6.1'
3
5
  end
data/lib/tweakphoeus.rb CHANGED
@@ -1,87 +1,46 @@
1
- require "tweakphoeus/version"
2
- require "tweakphoeus/user_agent"
3
- require "typhoeus"
1
+ # frozen_string_literal: true
2
+
3
+ require 'tweakphoeus/version'
4
+ require 'tweakphoeus/user_agent'
5
+ require 'tweakphoeus/cookie_jar'
6
+ require 'tweakphoeus/referer_list'
7
+ require 'typhoeus'
4
8
 
5
9
  module Tweakphoeus
10
+ # Http client class
6
11
  class Client
7
- attr_accessor :cookie_jar
8
- attr_accessor :base_headers
9
-
10
- def initialize(ua_systems: [:linux, :mac])
11
- @cookie_jar = {}
12
- @referer = [""]
13
- @base_headers = {
14
- "User-Agent" => UserAgent.random(systems: ua_systems),
15
- "Accept-Language" => "es-ES,es;q=0.9,en;q=0.8",
16
- "Accept-Encoding" => "",
17
- "Connection" => "keep-alive"
18
- }
12
+ attr_accessor :cookie_jar, :referer_list, :base_headers, :redirect
13
+ attr_reader :ssl_verifypeer
14
+
15
+ def initialize(ua_systems: %i[linux mac], ssl_verifypeer: true, redirect: false)
16
+ @cookie_jar = Tweakphoeus::CookieJar.new
17
+ @referer_list = Tweakphoeus::RefererList.new
18
+ @redirect = redirect
19
+ @base_headers = build_base_headers(ua_systems)
19
20
  @proxy = nil
20
21
  @proxyuserpwd = nil
21
- @ssl_verifypeer = true
22
- end
23
-
24
- def get(url, body: nil, params: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
25
- set_referer_from_headers(headers)
26
- http_request(url, body: body, params: params, headers: headers, redirect: redirect, method: :get, ssl_verifypeer: ssl_verifypeer)
27
- end
28
-
29
- def delete(url, body: nil, headers: nil, redirect: true, ssl_verifypeer: @ssl_verifypeer)
30
- set_referer_from_headers(headers)
31
- http_request(url, body: body, headers: headers, redirect: redirect, method: :delete, ssl_verifypeer: ssl_verifypeer)
32
- end
33
-
34
- def post(url, body: nil, params: nil, headers: nil, redirect: false, ssl_verifypeer: @ssl_verifypeer)
35
- set_referer_from_headers(headers)
36
- http_request(url, body: body, params: nil, headers: headers, redirect: redirect, method: :post, ssl_verifypeer: ssl_verifypeer)
37
- end
38
-
39
- def get_hide_inputs response
40
- #TODO
22
+ @ssl_verifypeer = ssl_verifypeer
41
23
  end
42
24
 
43
- def add_cookies host, key, value
44
- domain = get_domain(host)
45
- @cookie_jar[domain] ||= {}
46
- @cookie_jar[domain][key] = value
25
+ def get(url, body: nil, params: nil, headers: nil)
26
+ referer_from_headers(headers)
27
+ http_request(url, body: body, params: params, headers: headers, method: :get)
47
28
  end
48
29
 
49
- def get_domain domain
50
- domain.match(/([a-zA-Z0-9]+:\/\/|)([^\/]+)/)[2].gsub(/^\./,'')
30
+ def delete(url, body: nil, headers: nil)
31
+ referer_from_headers(headers)
32
+ http_request(url, body: body, headers: headers, method: :delete)
51
33
  end
52
34
 
53
- def push_referer url = ""
54
- @referer << url
55
- url
35
+ def post(url, body: nil, headers: nil)
36
+ referer_from_headers(headers)
37
+ http_request(url, body: body, headers: headers, method: :post)
56
38
  end
57
39
 
58
- def pop_referer
59
- @referer.pop
60
- end
40
+ def set_proxy(url, auth)
41
+ return unless auth
61
42
 
62
- def get_referer
63
- @referer.last
64
- end
65
-
66
- def cookie_string(url, headers={})
67
- domain = get_domain(url)
68
- headers ||= {}
69
- cookies = parse_cookie(headers["Cookie"])
70
-
71
- while domain != ""
72
- if @cookie_jar[domain]
73
- @cookie_jar[domain].each do |key, value|
74
- cookies[key] ||= value
75
- end
76
- end
77
- domain = domain.gsub(/^([^\.]+\.?)/, '')
78
- end
79
-
80
- cookies.map{ |key, value| "#{key}=#{value}" }.join('; ')
81
- end
82
-
83
- def set_proxy(url, auth = nil)
84
- @proxyuserpwd = "#{auth[:user]}:#{auth[:password]}" if auth.is_a?(Hash)
43
+ @proxyuserpwd = "#{auth[:user]}:#{auth[:password]}"
85
44
  @proxy = url
86
45
  end
87
46
 
@@ -92,94 +51,53 @@ module Tweakphoeus
92
51
 
93
52
  private
94
53
 
95
- def http_request(url, body: nil, params: nil, headers: nil, redirect: false, method: :get, ssl_verifypeer: @ssl_verifypeer)
96
- request_headers = merge_default_headers(headers)
97
- request_headers["Cookie"] = cookie_string(url, headers)
98
- request_headers["Referer"] = get_referer
99
- response = Typhoeus.send(method, url, body: body, params: params, headers: request_headers, proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
100
- obtain_cookies(response)
101
- set_referer(url) if method != :post
102
- if redirect && has_redirect?(response)
103
- if response.code != 307
104
- method = :get
105
- body = nil
106
- end
107
- response = http_request(redirect_url(response),
108
- body: body,
109
- headers: headers,
110
- redirect: redirect,
111
- method: method,
112
- ssl_verifypeer: ssl_verifypeer)
113
- end
114
- response
115
- end
54
+ def http_request(url, body: nil, params: nil, headers: nil, method: :get)
55
+ response = Typhoeus.send(method, url, body: body, params: params, headers: build_request_headers(headers),
56
+ proxy: @proxy, proxyuserpwd: @proxyuserpwd, ssl_verifypeer: ssl_verifypeer)
116
57
 
117
- def merge_default_headers headers
118
- headers ? @base_headers.merge(headers) : @base_headers
119
- end
58
+ @cookie_jar.obtain_cookies(response)
59
+ @referer_list.push_referer(url) if method != :post
120
60
 
121
- def obtain_cookies response
122
- set_cookies_field = response.headers["Set-Cookie"]
123
- return if set_cookies_field.nil?
124
- if set_cookies_field.is_a?(String)
125
- set_cookies_field = [set_cookies_field]
126
- end
61
+ return process_redirect(response) if redirect && redirect?(response)
127
62
 
128
- set_cookies_field.each do |cookie|
129
- key, value = cookie.match(/^([^=]+)=(.+)/).to_a[1..-1]
130
- domain = cookie.match(/Domain=([^;]+)/)
131
-
132
- if domain.nil?
133
- domain = get_domain(response.request.url)
134
- else
135
- domain = domain[1].gsub(/^\./,'')
136
- end
137
-
138
- value = value.split(';').first
139
- if value != "\"\""
140
- @cookie_jar[domain] ||= {}
141
- @cookie_jar[domain][key] = value
142
- else
143
- @cookie_jar[domain] ||= {}
144
- @cookie_jar[domain].delete(key)
145
- end
146
- end
63
+ response
147
64
  end
148
65
 
149
- def parse_cookie(string)
150
- cookies = {}
66
+ def build_request_headers(headers)
67
+ request_headers = merge_default_headers(headers)
68
+ request_headers['Cookie'] = @cookie_jar.cookie_string(url, headers)
69
+ request_headers['Referer'] = @referer_list.last_referer
70
+ end
151
71
 
152
- if string.is_a?(String)
153
- string.split(';').each do |part|
154
- key, value = part.split('=')
155
- key.strip!
156
- cookies[key.strip] = value if value && !["Domain","Path","domain","path"].include?(key)
157
- end
72
+ def process_redirect(response)
73
+ if response.code != 307
74
+ method = :get
75
+ body = nil
158
76
  end
159
77
 
160
- cookies
161
- end
162
-
163
- def has_redirect? response
164
- !redirect_url(response).nil?
78
+ http_request(redirect_url(response), body: body, headers: headers,
79
+ redirect: redirect, method: method, ssl_verifypeer: ssl_verifypeer)
165
80
  end
166
81
 
167
- def redirect_url response
168
- response.headers["Location"]
82
+ def build_base_headers(ua_systems)
83
+ {
84
+ 'User-Agent' => UserAgent.random(systems: ua_systems),
85
+ 'Accept-Language' => 'es-ES,es;q=0.9,en;q=0.8',
86
+ 'Accept-Encoding' => '',
87
+ 'Connection' => 'keep-alive'
88
+ }
169
89
  end
170
90
 
171
- def purge_bad_cookies cookies
172
- cookies.reject{|e| e.first.last=="\"\""}
91
+ def merge_default_headers(headers)
92
+ headers ? @base_headers.merge(headers) : @base_headers
173
93
  end
174
94
 
175
- def set_referer_from_headers headers
176
- if headers && headers["Referer"].is_a?(String)
177
- @referer.last.replace headers["Referer"]
178
- end
95
+ def redirect?(response)
96
+ !redirect_url(response).nil?
179
97
  end
180
98
 
181
- def set_referer url
182
- @referer.last.replace url
99
+ def redirect_url(response)
100
+ response.headers['Location']
183
101
  end
184
102
  end
185
103
  end
data/tweakphoeus.gemspec CHANGED
@@ -1,34 +1,34 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
3
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require 'tweakphoeus/version'
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'lib/tweakphoeus/version'
5
4
 
6
5
  Gem::Specification.new do |spec|
7
- spec.name = "tweakphoeus"
6
+ spec.name = 'tweakphoeus'
8
7
  spec.version = Tweakphoeus::VERSION
9
- spec.authors = ["David Martin Garcia"]
10
- spec.email = ["base.dks@gmail.com"]
8
+ spec.authors = ['David Martin Garcia']
9
+ spec.email = ['davidmartingarcia@gmail.com']
11
10
 
12
- spec.summary = %q{Typhoeus on steroids.}
13
- spec.description = %q{Typhoeus wrapper with new pretty things: cookie handler for example.}
14
- spec.homepage = "https://github.com/basestylo/Tweakphoeus/"
15
- spec.license = "MIT"
11
+ spec.summary = 'Typhoeus on steroids.'
12
+ spec.description = 'Typhoeus wrapper with some extras.'
13
+ spec.homepage = 'https://github.com/basestylo/Tweakphoeus/'
14
+ spec.license = 'MIT'
15
+ spec.required_ruby_version = '>= 2.6.0'
16
16
 
17
- # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
18
- # delete this section to allow pushing this gem to any host.
19
- if spec.respond_to?(:metadata)
20
- spec.metadata['allowed_push_host'] = "https://rubygems.org"
21
- else
22
- raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
23
- end
17
+ spec.metadata['homepage_uri'] = spec.homepage
18
+ spec.metadata['source_code_uri'] = 'https://github.com/dmartingarcia/Tweakphoeus'
19
+ spec.metadata['changelog_uri'] = 'https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md'
24
20
 
25
- spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
26
- spec.bindir = "exe"
27
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
- spec.require_paths = ["lib"]
21
+ # Specify which files should be added to the gem when it is released.
22
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
23
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
24
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
25
+ end
26
+ spec.bindir = 'exe'
27
+ spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
28
+ spec.require_paths = ['lib']
29
29
 
30
- spec.add_dependency 'typhoeus', '~> 1.3', '>= 1.3.0'
31
- spec.add_development_dependency "bundler", "~> 1.10"
32
- spec.add_development_dependency "rake", "~> 10.0"
33
- spec.add_development_dependency "rspec", "~> 3.0", "> 3.0.0"
30
+ spec.add_dependency 'typhoeus', '~> 1.4'
31
+ spec.add_development_dependency 'bundler', '~> 2.2'
32
+ spec.add_development_dependency 'rake', '~> 13.0'
33
+ spec.add_development_dependency 'rspec', '~> 3.10'
34
34
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tweakphoeus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.5
4
+ version: 0.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - David Martin Garcia
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-05-22 00:00:00.000000000 Z
11
+ date: 2022-04-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: typhoeus
@@ -16,78 +16,71 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.3'
20
- - - ">="
21
- - !ruby/object:Gem::Version
22
- version: 1.3.0
19
+ version: '1.4'
23
20
  type: :runtime
24
21
  prerelease: false
25
22
  version_requirements: !ruby/object:Gem::Requirement
26
23
  requirements:
27
24
  - - "~>"
28
25
  - !ruby/object:Gem::Version
29
- version: '1.3'
30
- - - ">="
31
- - !ruby/object:Gem::Version
32
- version: 1.3.0
26
+ version: '1.4'
33
27
  - !ruby/object:Gem::Dependency
34
28
  name: bundler
35
29
  requirement: !ruby/object:Gem::Requirement
36
30
  requirements:
37
31
  - - "~>"
38
32
  - !ruby/object:Gem::Version
39
- version: '1.10'
33
+ version: '2.2'
40
34
  type: :development
41
35
  prerelease: false
42
36
  version_requirements: !ruby/object:Gem::Requirement
43
37
  requirements:
44
38
  - - "~>"
45
39
  - !ruby/object:Gem::Version
46
- version: '1.10'
40
+ version: '2.2'
47
41
  - !ruby/object:Gem::Dependency
48
42
  name: rake
49
43
  requirement: !ruby/object:Gem::Requirement
50
44
  requirements:
51
45
  - - "~>"
52
46
  - !ruby/object:Gem::Version
53
- version: '10.0'
47
+ version: '13.0'
54
48
  type: :development
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
51
  requirements:
58
52
  - - "~>"
59
53
  - !ruby/object:Gem::Version
60
- version: '10.0'
54
+ version: '13.0'
61
55
  - !ruby/object:Gem::Dependency
62
56
  name: rspec
63
57
  requirement: !ruby/object:Gem::Requirement
64
58
  requirements:
65
59
  - - "~>"
66
60
  - !ruby/object:Gem::Version
67
- version: '3.0'
68
- - - ">"
69
- - !ruby/object:Gem::Version
70
- version: 3.0.0
61
+ version: '3.10'
71
62
  type: :development
72
63
  prerelease: false
73
64
  version_requirements: !ruby/object:Gem::Requirement
74
65
  requirements:
75
66
  - - "~>"
76
67
  - !ruby/object:Gem::Version
77
- version: '3.0'
78
- - - ">"
79
- - !ruby/object:Gem::Version
80
- version: 3.0.0
81
- description: 'Typhoeus wrapper with new pretty things: cookie handler for example.'
68
+ version: '3.10'
69
+ description: Typhoeus wrapper with some extras.
82
70
  email:
83
- - base.dks@gmail.com
71
+ - davidmartingarcia@gmail.com
84
72
  executables: []
85
73
  extensions: []
86
74
  extra_rdoc_files: []
87
75
  files:
76
+ - ".github/workflows/auto-tag.yaml"
77
+ - ".github/workflows/main.yml"
78
+ - ".github/workflows/on-pull-request.yaml"
79
+ - ".github/workflows/on_new_tag.yaml"
80
+ - ".github/workflows/version-forget-me-not.yaml"
88
81
  - ".gitignore"
89
82
  - ".rspec"
90
- - ".travis.yml"
83
+ - ".rubocop.yml"
91
84
  - CHANGELOG.md
92
85
  - CODE_OF_CONDUCT.md
93
86
  - Gemfile
@@ -97,6 +90,9 @@ files:
97
90
  - bin/console
98
91
  - bin/setup
99
92
  - lib/tweakphoeus.rb
93
+ - lib/tweakphoeus/cookie_jar.rb
94
+ - lib/tweakphoeus/helpers.rb
95
+ - lib/tweakphoeus/referer_list.rb
100
96
  - lib/tweakphoeus/user_agent.rb
101
97
  - lib/tweakphoeus/version.rb
102
98
  - tweakphoeus.gemspec
@@ -104,7 +100,9 @@ homepage: https://github.com/basestylo/Tweakphoeus/
104
100
  licenses:
105
101
  - MIT
106
102
  metadata:
107
- allowed_push_host: https://rubygems.org
103
+ homepage_uri: https://github.com/basestylo/Tweakphoeus/
104
+ source_code_uri: https://github.com/dmartingarcia/Tweakphoeus
105
+ changelog_uri: https://github.com/dmartingarcia/Tweakphoeus/blob/master/CHANGELOG.md
108
106
  post_install_message:
109
107
  rdoc_options: []
110
108
  require_paths:
@@ -113,15 +111,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
113
111
  requirements:
114
112
  - - ">="
115
113
  - !ruby/object:Gem::Version
116
- version: '0'
114
+ version: 2.6.0
117
115
  required_rubygems_version: !ruby/object:Gem::Requirement
118
116
  requirements:
119
117
  - - ">="
120
118
  - !ruby/object:Gem::Version
121
119
  version: '0'
122
120
  requirements: []
123
- rubyforge_project:
124
- rubygems_version: 2.7.3
121
+ rubygems_version: 3.1.2
125
122
  signing_key:
126
123
  specification_version: 4
127
124
  summary: Typhoeus on steroids.
data/.travis.yml DELETED
@@ -1,4 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.2.3
4
- before_install: gem install bundler -v 1.10.6