stealth_browser_automation 0.0.11 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/browserfactory.rb +121 -11
- data/lib/proxy.rb +2 -2
- metadata +3 -63
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c572d341b87a3b41503bba5fe37f805133d48ebc
|
4
|
+
data.tar.gz: c5e1a0d4d749cfe28ebfab77b3e0d82344588126
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f880759a9c4ec4e9354eb42c9dabf4122ccb5f5ab5f86c69ad0793528a766916bc49c8513d9e1b85e31b8520ade556b59b43810ed76d1b167567dc40469a63d
|
7
|
+
data.tar.gz: fd4321ed247d53561f45f38176748539fe1cfb78dd4be4ae9c780bc733cf75af1eb8e6922af33628a24348384bccd0e5c99b069be91136029962bcb10e7fbe48
|
data/lib/browserfactory.rb
CHANGED
@@ -1,12 +1,112 @@
|
|
1
1
|
require 'watir-webdriver'
|
2
2
|
require "pathname"
|
3
3
|
|
4
|
+
# **ABSTRACT**
|
5
|
+
#
|
6
|
+
# PampaBrowser should pull a screenshot every N seconds, from a thread.
|
7
|
+
# But can't operate the same browser from more than 1 thread.
|
8
|
+
# More information here:
|
9
|
+
# https://stackoverflow.com/questions/28093347/is-selenium-webdriver-thread-safe
|
10
|
+
#
|
11
|
+
# Should use a lockfile.
|
12
|
+
# More information here:
|
13
|
+
# https://bibwild.wordpress.com/2015/01/15/ruby-threads-gotcha-with-local-vars-and-shared-state/
|
14
|
+
# https://stackoverflow.com/questions/55895952/ruby-how-do-i-share-a-global-variable-amongst-threads-that-are-running-an-objec
|
15
|
+
#
|
16
|
+
# So, I should overwrite all methods of the Selenium::Browser, Selenium::Element, and also Watir::Wait
|
17
|
+
# More information here:
|
18
|
+
# https://stackoverflow.com/questions/6966708/how-to-add-statements-to-an-existing-method-definition-in-ruby
|
19
|
+
# https://stackoverflow.com/questions/44577888/ruby-alias-method-for-module-static-method
|
20
|
+
#
|
21
|
+
#
|
22
|
+
# **The "frequency-based-streaming" model**
|
23
|
+
#
|
24
|
+
# At this moment, I couldn't overwrite Watir::Wait's methods.
|
25
|
+
# At this moment, PampaBrowser is simple sending the screenshot when some particular methods has been called,
|
26
|
+
# but this appreach is not updating the screen when the website load new contenct thru AJAX with not need of
|
27
|
+
# and user interaction.
|
28
|
+
#
|
29
|
+
|
30
|
+
module Selenium
|
31
|
+
module WebDriver
|
32
|
+
class Element
|
33
|
+
alias_method :old_click, :click
|
34
|
+
|
35
|
+
def click()
|
36
|
+
old_click
|
37
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
38
|
+
end # def click
|
39
|
+
|
40
|
+
end # class Element
|
41
|
+
end # module WebDriver
|
42
|
+
end # module Selenium
|
43
|
+
|
4
44
|
module BlackStack
|
5
45
|
|
6
46
|
class PampaBrowser < Watir::Browser
|
47
|
+
|
48
|
+
# TODO: enable this when you move to the "frequency-based-streaming" model
|
49
|
+
#PAMPA_BROWSER_CHANNEL_LOCK_FILENAME = './browserfactory.channel.%PRFILE_NAME%.lock' # manejo de concurrencia en la creación de browsers
|
50
|
+
#attr_accessor :lockfile
|
7
51
|
|
8
52
|
attr_accessor :proxy, :lnuser, :agent_name, :profile_name
|
9
|
-
|
53
|
+
|
54
|
+
def lock_channel()
|
55
|
+
self.lockfile.flock(File::LOCK_EX)
|
56
|
+
end
|
57
|
+
|
58
|
+
def release_channel()
|
59
|
+
self.lockfile.flock(File::LOCK_UN)
|
60
|
+
end
|
61
|
+
|
62
|
+
def initialize(driver)
|
63
|
+
# TODO: enable this when you move to the "frequency-based-streaming" model
|
64
|
+
#fname = PAMPA_BROWSER_CHANNEL_LOCK_FILENAME.gsub('%PRFILE_NAME%', self.profile_name.to_s)
|
65
|
+
#self.lockfile = File.open(fname,"w")
|
66
|
+
|
67
|
+
#
|
68
|
+
super(driver)
|
69
|
+
|
70
|
+
# TODO: enable this when you move to the "frequency-based-streaming" model
|
71
|
+
=begin
|
72
|
+
# Can't operate the same browser from more than 1 thread.
|
73
|
+
# More information here:
|
74
|
+
# https://stackoverflow.com/questions/28093347/is-selenium-webdriver-thread-safe
|
75
|
+
#
|
76
|
+
# https://bibwild.wordpress.com/2015/01/15/ruby-threads-gotcha-with-local-vars-and-shared-state/
|
77
|
+
# https://stackoverflow.com/questions/55895952/ruby-how-do-i-share-a-global-variable-amongst-threads-that-are-running-an-objec
|
78
|
+
#
|
79
|
+
t0 = Thread.new {
|
80
|
+
minimum_enlapsed_seconds = 5.to_f
|
81
|
+
while true
|
82
|
+
start_time = Time.now
|
83
|
+
t1 = Thread.new(self) do |t_browser|
|
84
|
+
#while true
|
85
|
+
begin
|
86
|
+
self.lock_channel
|
87
|
+
sleep(1) # para esperar a que se ejecute la sentencia t1.join debajo
|
88
|
+
t_browser.push_screenshot
|
89
|
+
self.release_channel
|
90
|
+
rescue => e
|
91
|
+
# Communication with the chrome driver errors may occur.
|
92
|
+
# If an exception happens here and it is not catched, then
|
93
|
+
# then the thread will stop workng and the streaming will
|
94
|
+
# stop.
|
95
|
+
end
|
96
|
+
#end # while true
|
97
|
+
end # Thread.new
|
98
|
+
t1.join
|
99
|
+
end_time = Time.now
|
100
|
+
elapsed_seconds = (end_time - start_time).to_f
|
101
|
+
if (elapsed_seconds < minimum_enlapsed_seconds)
|
102
|
+
sleep_seconds = minimum_enlapsed_seconds - elapsed_seconds
|
103
|
+
sleep(sleep_seconds.to_i + 1)
|
104
|
+
end # if
|
105
|
+
end # while true
|
106
|
+
} # Thread.new
|
107
|
+
=end
|
108
|
+
end # def initialize
|
109
|
+
|
10
110
|
def notify(method)
|
11
111
|
url = "#{BlackStack::Pampa::api_protocol}://#{PROCESS.ws_url}:#{PROCESS.ws_port}/api1.3/pampa/browser/notify.json"
|
12
112
|
res = BlackStack::Netting::call_post(url, {
|
@@ -25,7 +125,7 @@ module BlackStack
|
|
25
125
|
raise "Error Tracing BrowserActivity: #{parsed['status']}"
|
26
126
|
end
|
27
127
|
end
|
28
|
-
|
128
|
+
|
29
129
|
def push_screenshot(filename=nil)
|
30
130
|
filename = "#{PROCESS.fullWorkerName}.png" if filename.nil?
|
31
131
|
BrowserFactory::screenshot(filename)
|
@@ -40,6 +140,7 @@ module BlackStack
|
|
40
140
|
url = "#{BlackStack::Pampa::api_protocol}://#{PROCESS.ws_url}:#{PROCESS.ws_port}/api1.3/pampa/browser/screenshot.json"
|
41
141
|
res = RestClient::Request.execute(
|
42
142
|
:api_key => BlackStack::Pampa::api_key,
|
143
|
+
:id_client => PROCESS.id_client,
|
43
144
|
:verify_ssl => false,
|
44
145
|
:url => url,
|
45
146
|
:method => :post,
|
@@ -64,73 +165,80 @@ module BlackStack
|
|
64
165
|
def back
|
65
166
|
self.notify("back")
|
66
167
|
super
|
67
|
-
|
168
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
68
169
|
end
|
69
170
|
|
70
171
|
def forward
|
71
172
|
self.notify("forward")
|
72
173
|
super
|
73
|
-
|
174
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
74
175
|
end
|
75
176
|
|
76
177
|
def execute_script(script, *args)
|
77
178
|
self.notify("execute_script")
|
78
179
|
super
|
79
|
-
|
180
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
80
181
|
end
|
81
182
|
|
82
183
|
def goto(url)
|
83
184
|
self.notify("goto")
|
84
185
|
super
|
85
|
-
|
186
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
86
187
|
end
|
87
188
|
|
88
189
|
def screenshot
|
89
190
|
self.notify("screenshot")
|
90
191
|
super
|
192
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
91
193
|
end
|
92
194
|
|
93
195
|
def reset!
|
94
196
|
self.notify("reset!")
|
95
197
|
super
|
198
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
96
199
|
end
|
97
200
|
|
98
201
|
def refresh
|
99
202
|
self.notify("refresh")
|
100
203
|
super
|
101
|
-
|
204
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
102
205
|
end
|
103
206
|
|
104
207
|
def inspect
|
105
208
|
self.notify("inspect")
|
106
209
|
super
|
210
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
107
211
|
end
|
108
212
|
|
109
|
-
=begin # se da de baja, porque incremente el # de llamadas a la API de 800/hora a 5000/hora, y eso requiere mas infraestructura
|
110
213
|
def send_keys(*args)
|
111
|
-
|
214
|
+
# no se notifica el evento, porque incremente el # de llamadas a la API de 800/hora a 5000/hora, y eso requiere mas infraestructura
|
215
|
+
#self.notify("send_keys")
|
112
216
|
super
|
217
|
+
#BlackStack::BrowserFactory.browser.push_screenshot
|
113
218
|
end
|
114
|
-
=end
|
115
219
|
|
116
220
|
def text
|
117
221
|
self.notify("text")
|
118
222
|
super
|
223
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
119
224
|
end
|
120
225
|
|
121
226
|
def title
|
122
227
|
self.notify("title")
|
123
228
|
super
|
229
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
124
230
|
end
|
125
231
|
|
126
232
|
def url
|
127
233
|
self.notify("url")
|
128
234
|
super
|
235
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
129
236
|
end
|
130
237
|
|
131
238
|
def wait(timeout = 5)
|
132
239
|
self.notify("wait")
|
133
240
|
super
|
241
|
+
BlackStack::BrowserFactory.browser.push_screenshot
|
134
242
|
end
|
135
243
|
end # class PampaBrowser
|
136
244
|
|
@@ -187,6 +295,9 @@ module BlackStack
|
|
187
295
|
@@pid = nil
|
188
296
|
@@profile_name = nil
|
189
297
|
|
298
|
+
def self.browser()
|
299
|
+
@@browser
|
300
|
+
end
|
190
301
|
|
191
302
|
def self.lockProfileList()
|
192
303
|
# @@fd_profile.flock(File::LOCK_EX)
|
@@ -203,7 +314,6 @@ module BlackStack
|
|
203
314
|
=end
|
204
315
|
end
|
205
316
|
|
206
|
-
|
207
317
|
# NOTA: esta lista debe estar permitida por LinkedIn. De caso contrario, aparecera el mensaje "Upgrad your browser"
|
208
318
|
# Se puede obtener una lista actualizada de este sitio: https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
|
209
319
|
@@arAgents = [
|
data/lib/proxy.rb
CHANGED
@@ -60,7 +60,7 @@ module BlackStack
|
|
60
60
|
end
|
61
61
|
|
62
62
|
# TODO: Reemplazar el parametro records_before_long_sleep por un registro en la tabla PARAMS
|
63
|
-
def endJob(
|
63
|
+
def endJob(id_client, result, description, records_before_long_sleep)
|
64
64
|
# => #464
|
65
65
|
=begin
|
66
66
|
DB.execute("UPDATE proxy SET reservation_end_time=GETDATE(), reservation_result='#{result.to_s}', reservation_description='#{description.to_s}' WHERE [id]='#{self.id}'")
|
@@ -68,7 +68,7 @@ module BlackStack
|
|
68
68
|
DB.execute(
|
69
69
|
"EXEC dbo.endProxyJob " +
|
70
70
|
"'#{self.id}', " +
|
71
|
-
"'#{
|
71
|
+
"'#{id_client}', " +
|
72
72
|
"#{result.to_s}, " +
|
73
73
|
"'#{description.to_s}', " +
|
74
74
|
"#{records_before_long_sleep.to_s} "
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: stealth_browser_automation
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Leandro Daniel Sardi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mini_magick
|
@@ -131,47 +131,7 @@ dependencies:
|
|
131
131
|
- !ruby/object:Gem::Version
|
132
132
|
version: 4.28.0
|
133
133
|
- !ruby/object:Gem::Dependency
|
134
|
-
name:
|
135
|
-
requirement: !ruby/object:Gem::Requirement
|
136
|
-
requirements:
|
137
|
-
- - "~>"
|
138
|
-
- !ruby/object:Gem::Version
|
139
|
-
version: 0.0.20
|
140
|
-
- - ">="
|
141
|
-
- !ruby/object:Gem::Version
|
142
|
-
version: 0.0.20
|
143
|
-
type: :runtime
|
144
|
-
prerelease: false
|
145
|
-
version_requirements: !ruby/object:Gem::Requirement
|
146
|
-
requirements:
|
147
|
-
- - "~>"
|
148
|
-
- !ruby/object:Gem::Version
|
149
|
-
version: 0.0.20
|
150
|
-
- - ">="
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: 0.0.20
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
|
-
name: simple_cloud_logging
|
155
|
-
requirement: !ruby/object:Gem::Requirement
|
156
|
-
requirements:
|
157
|
-
- - "~>"
|
158
|
-
- !ruby/object:Gem::Version
|
159
|
-
version: 1.1.16
|
160
|
-
- - ">="
|
161
|
-
- !ruby/object:Gem::Version
|
162
|
-
version: 1.1.16
|
163
|
-
type: :runtime
|
164
|
-
prerelease: false
|
165
|
-
version_requirements: !ruby/object:Gem::Requirement
|
166
|
-
requirements:
|
167
|
-
- - "~>"
|
168
|
-
- !ruby/object:Gem::Version
|
169
|
-
version: 1.1.16
|
170
|
-
- - ">="
|
171
|
-
- !ruby/object:Gem::Version
|
172
|
-
version: 1.1.16
|
173
|
-
- !ruby/object:Gem::Dependency
|
174
|
-
name: simple_command_line_parser
|
134
|
+
name: pampa_workers
|
175
135
|
requirement: !ruby/object:Gem::Requirement
|
176
136
|
requirements:
|
177
137
|
- - "~>"
|
@@ -190,26 +150,6 @@ dependencies:
|
|
190
150
|
- - ">="
|
191
151
|
- !ruby/object:Gem::Version
|
192
152
|
version: 1.1.1
|
193
|
-
- !ruby/object:Gem::Dependency
|
194
|
-
name: pampa_workers
|
195
|
-
requirement: !ruby/object:Gem::Requirement
|
196
|
-
requirements:
|
197
|
-
- - "~>"
|
198
|
-
- !ruby/object:Gem::Version
|
199
|
-
version: 0.0.39
|
200
|
-
- - ">="
|
201
|
-
- !ruby/object:Gem::Version
|
202
|
-
version: 0.0.39
|
203
|
-
type: :runtime
|
204
|
-
prerelease: false
|
205
|
-
version_requirements: !ruby/object:Gem::Requirement
|
206
|
-
requirements:
|
207
|
-
- - "~>"
|
208
|
-
- !ruby/object:Gem::Version
|
209
|
-
version: 0.0.39
|
210
|
-
- - ">="
|
211
|
-
- !ruby/object:Gem::Version
|
212
|
-
version: 0.0.39
|
213
153
|
description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/stealth_browser_automation.'
|
214
154
|
email: leandro.sardi@expandedventure.com
|
215
155
|
executables: []
|