pampa_workers 1.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/basedivision.rb +5 -0
- data/lib/baseworker.rb +5 -0
- data/lib/client.rb +189 -0
- data/lib/division.rb +46 -0
- data/lib/login.rb +7 -0
- data/lib/mybotprocess.rb +348 -0
- data/lib/mychildprocess.rb +9 -0
- data/lib/mycrawlprocess.rb +49 -0
- data/lib/mylocalprocess.rb +170 -0
- data/lib/myparentprocess.rb +142 -0
- data/lib/myprocess.rb +265 -0
- data/lib/myremoteprocess.rb +138 -0
- data/lib/pampa-local.rb +41 -0
- data/lib/pampa_workers.rb +257 -0
- data/lib/params.rb +50 -0
- data/lib/remotedivision.rb +8 -0
- data/lib/remoteworker.rb +8 -0
- data/lib/role.rb +8 -0
- data/lib/timezone.rb +151 -0
- data/lib/user.rb +25 -0
- data/lib/userdivision.rb +5 -0
- data/lib/userrole.rb +8 -0
- data/lib/worker.rb +154 -0
- metadata +169 -0
@@ -0,0 +1,138 @@
|
|
1
|
+
module BlackStack
|
2
|
+
|
3
|
+
# no maneja conexion a la base de datos.
|
4
|
+
# ejecuta un loop mientras el metodo canRun? retorne true.
|
5
|
+
class MyRemoteProcess < BlackStack::MyChildProcess
|
6
|
+
#
|
7
|
+
attr_accessor :worker
|
8
|
+
|
9
|
+
# update worker configuration in the division
|
10
|
+
def updateWorker()
|
11
|
+
# creo un remote worker que manejare en este proceso remote
|
12
|
+
self.worker = BlackStack::RemoteWorker.new
|
13
|
+
# me notifico a la central. obtengo asignacion si ya la tenia
|
14
|
+
# y vuelco la configuracion al remote worker
|
15
|
+
url = "#{BlackStack::Pampa::api_url}/api1.3/pampa/get.json"
|
16
|
+
res = BlackStack::Netting::call_post(url, {
|
17
|
+
'api_key' => BlackStack::Pampa::api_key,
|
18
|
+
'name' => self.fullWorkerName }.merge( BlackStack::RemoteHost.new.poll )
|
19
|
+
)
|
20
|
+
parsed = JSON.parse(res.body)
|
21
|
+
if (parsed['status'] != BlackStack::Netting::SUCCESS)
|
22
|
+
raise parsed['status'].to_s
|
23
|
+
else
|
24
|
+
self.worker.id = parsed['id']
|
25
|
+
self.worker.assigned_process = parsed['assigned_process']
|
26
|
+
self.worker.id_client = parsed['id_client']
|
27
|
+
self.worker.id_division = parsed['id_division']
|
28
|
+
self.worker.division_name = parsed['division_name']
|
29
|
+
self.worker.ws_url = parsed['ws_url']
|
30
|
+
self.worker.ws_port = parsed['ws_port']
|
31
|
+
self.worker.division = BlackStack::RemoteDivision.new
|
32
|
+
self.worker.division.name = parsed['division_name']
|
33
|
+
end
|
34
|
+
# llamo al metodo de la clase padre que reporta la configuracion a
|
35
|
+
# la division del worker
|
36
|
+
self.set(parsed['assigned_process'], parsed['id_client'])
|
37
|
+
end
|
38
|
+
|
39
|
+
#
|
40
|
+
def run()
|
41
|
+
super
|
42
|
+
|
43
|
+
# creo el objeto logger
|
44
|
+
self.logger = RemoteLogger.new(
|
45
|
+
"#{self.fullWorkerName}.log",
|
46
|
+
BlackStack::Pampa::api_protocol,
|
47
|
+
BlackStack::Pampa::api_domain,
|
48
|
+
BlackStack::Pampa::api_port,
|
49
|
+
BlackStack::Pampa::api_key,
|
50
|
+
self.id_client # ID of the client that has this thread assigned
|
51
|
+
)
|
52
|
+
|
53
|
+
logger.log "Remote process is alive!"
|
54
|
+
|
55
|
+
# actualiza parametros de la central
|
56
|
+
logger.logs "Update from central (1-remote)... "
|
57
|
+
self.get
|
58
|
+
logger.done
|
59
|
+
|
60
|
+
# actualizo los datos de este worker (parent process)
|
61
|
+
logger.logs "Update worker (1-remote)... "
|
62
|
+
self.updateWorker
|
63
|
+
logger.done
|
64
|
+
|
65
|
+
# actualizo los datos de este worker (parent process)
|
66
|
+
logger.logs "Switch logger id_client (log folder may change)... "
|
67
|
+
self.logger.id_client = self.id_client
|
68
|
+
logger.done
|
69
|
+
|
70
|
+
while (self.canRun?)
|
71
|
+
|
72
|
+
# reseteo en contador nested del logger
|
73
|
+
self.logger.reset()
|
74
|
+
|
75
|
+
# announcing my in the log
|
76
|
+
logger.log "Going to Run Remote"
|
77
|
+
logger.log "Process: #{self.assigned_process.to_s}."
|
78
|
+
logger.log "Client: #{(self.id_client.to_s.size==0)? 'n/a' : self.id_client.to_s}."
|
79
|
+
|
80
|
+
# obtengo la hora de inicio
|
81
|
+
start_time = Time.now
|
82
|
+
|
83
|
+
begin
|
84
|
+
# libero recursos
|
85
|
+
logger.logs "Release resources... "
|
86
|
+
GC.start
|
87
|
+
#DB.disconnect
|
88
|
+
logger.done
|
89
|
+
|
90
|
+
# envia senial a la central.
|
91
|
+
# si tiene asignada una division, envia senial a la division.
|
92
|
+
logger.logs "Ping... "
|
93
|
+
self.ping()
|
94
|
+
logger.done
|
95
|
+
|
96
|
+
# envia senial a la central.
|
97
|
+
# si tiene asignada una division, envia senial a la division.
|
98
|
+
logger.logs "Notify to Division... "
|
99
|
+
self.notify()
|
100
|
+
logger.done
|
101
|
+
|
102
|
+
# corro el procesamiento
|
103
|
+
self.process(ARGV)
|
104
|
+
|
105
|
+
rescue => e
|
106
|
+
puts ""
|
107
|
+
logger.log "Remote Process Error: " + e.to_s + "\r\n" + e.backtrace.join("\r\n").to_s
|
108
|
+
end
|
109
|
+
|
110
|
+
# actualiza parametros de la central
|
111
|
+
logger.logs "Update from central (2)... "
|
112
|
+
self.get
|
113
|
+
logger.done
|
114
|
+
|
115
|
+
# actualizo los datos de este worker (parent process)
|
116
|
+
logger.logs "Update worker (2)... "
|
117
|
+
self.updateWorker
|
118
|
+
logger.done
|
119
|
+
|
120
|
+
# sleep
|
121
|
+
logger.logs "Sleep... "
|
122
|
+
self.doSleep(start_time)
|
123
|
+
logger.done
|
124
|
+
|
125
|
+
logger.log "-------------------------------------------"
|
126
|
+
|
127
|
+
GC.start
|
128
|
+
#DB.disconnect
|
129
|
+
|
130
|
+
end # main while
|
131
|
+
|
132
|
+
#
|
133
|
+
logger.log self.whyCantRun()
|
134
|
+
|
135
|
+
end # run
|
136
|
+
end # class MyRemoteProcess
|
137
|
+
|
138
|
+
end # module BlackStack
|
data/lib/pampa-local.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative './division'
|
2
|
+
require_relative './params'
|
3
|
+
require_relative './worker'
|
4
|
+
require_relative './client'
|
5
|
+
require_relative './timezone'
|
6
|
+
require_relative './user'
|
7
|
+
require_relative './login'
|
8
|
+
require_relative './role'
|
9
|
+
require_relative './userdivision'
|
10
|
+
require_relative './userrole'
|
11
|
+
|
12
|
+
# funciones auxiliares
|
13
|
+
def guid()
|
14
|
+
DB['SELECT NEWID() AS [id]'].map(:id)[0]
|
15
|
+
end
|
16
|
+
|
17
|
+
def now()
|
18
|
+
# La llamada a GETDATE() desde ODBC no retorna precision de milisegundos, que es necesaria para los registros de log.
|
19
|
+
# La llamada a SYSDATETIME() retorna un valor de alta precision que no es compatible para pegar en los campos del tipo DATATIME.
|
20
|
+
# La linea de abajo obtiene la hora en formato de SYSDATE y le trunca los ultimos digitos para hacer que el valor sea compatible con los campos DATETIME.
|
21
|
+
(DB['SELECT SYSDATETIME() AS [now]'].map(:now)[0]).to_s[0..22]
|
22
|
+
end
|
23
|
+
|
24
|
+
def diff(unit, t0, t1)
|
25
|
+
# La llamada a GETDATE() desde ODBC no retorna precision de milisegundos, que es necesaria para los registros de log.
|
26
|
+
# La llamada a SYSDATETIME() retorna un valor de alta precision que no es compatible para pegar en los campos del tipo DATATIME.
|
27
|
+
# La linea de abajo obtiene la hora en formato de SYSDATE y le trunca los ultimos digitos para hacer que el valor sea compatible con los campos DATETIME.
|
28
|
+
(DB["SELECT DATEDIFF(#{unit}, '#{t0.to_s}', '#{t1.to_s}') AS [diff]"].map(:diff)[0]).to_i
|
29
|
+
end
|
30
|
+
|
31
|
+
def before(n) # n minutes
|
32
|
+
DB["SELECT DATEADD(mi, -#{n.to_s}, GETDATE()) AS [now]"].map(:now)[0].to_s[0..22]
|
33
|
+
end
|
34
|
+
|
35
|
+
def monthsFromNow(n) # n months
|
36
|
+
DB["SELECT DATEADD(mm, +#{n.to_s}, GETDATE()) AS [now]"].map(:now)[0].to_s[0..22]
|
37
|
+
end
|
38
|
+
|
39
|
+
def daysFromNow(n) # n days
|
40
|
+
DB["SELECT DATEADD(dd, +#{n.to_s}, GETDATE()) AS [now]"].map(:now)[0].to_s[0..22]
|
41
|
+
end
|
@@ -0,0 +1,257 @@
|
|
1
|
+
require 'simple_host_monitoring'
|
2
|
+
require 'socket'
|
3
|
+
require 'time'
|
4
|
+
require 'uri'
|
5
|
+
require 'net/http'
|
6
|
+
require 'json'
|
7
|
+
require 'openssl'
|
8
|
+
require 'tiny_tds'
|
9
|
+
require 'sequel'
|
10
|
+
|
11
|
+
require_relative './baseworker'
|
12
|
+
require_relative './basedivision'
|
13
|
+
|
14
|
+
require_relative './remoteworker'
|
15
|
+
require_relative './remotedivision'
|
16
|
+
|
17
|
+
require_relative './myprocess'
|
18
|
+
require_relative './mychildprocess'
|
19
|
+
require_relative './mylocalprocess'
|
20
|
+
require_relative './myparentprocess'
|
21
|
+
require_relative './myremoteprocess'
|
22
|
+
require_relative './mybotprocess'
|
23
|
+
require_relative './mycrawlprocess'
|
24
|
+
require_relative './remoteworker'
|
25
|
+
require_relative './remotedivision'
|
26
|
+
|
27
|
+
module BlackStack
|
28
|
+
|
29
|
+
module Pampa
|
30
|
+
|
31
|
+
SLEEP_SECONDS = 10
|
32
|
+
|
33
|
+
#
|
34
|
+
@@division_name = nil
|
35
|
+
|
36
|
+
def self.division_name()
|
37
|
+
@@division_name
|
38
|
+
end
|
39
|
+
|
40
|
+
#
|
41
|
+
def self.set_division_name(s)
|
42
|
+
@@division_name = s
|
43
|
+
end
|
44
|
+
|
45
|
+
# Api-key of the client who will be the owner of a process.
|
46
|
+
@@api_key = nil
|
47
|
+
|
48
|
+
def self.api_key()
|
49
|
+
@@api_key
|
50
|
+
end
|
51
|
+
|
52
|
+
# Protocol, HTTP or HTTPS, of the BlackStack server where this process will be registered.
|
53
|
+
@@api_protocol = nil
|
54
|
+
|
55
|
+
#
|
56
|
+
def self.api_protocol
|
57
|
+
@@api_protocol
|
58
|
+
end
|
59
|
+
|
60
|
+
# Domain of the BlackStack server where this process will be registered.
|
61
|
+
@@api_domain = nil
|
62
|
+
|
63
|
+
#
|
64
|
+
def self.api_domain
|
65
|
+
@@api_domain
|
66
|
+
end
|
67
|
+
|
68
|
+
# Port of the BlackStack server where this process will be registered.
|
69
|
+
@@api_port = nil
|
70
|
+
|
71
|
+
#
|
72
|
+
def self.api_port
|
73
|
+
@@api_port
|
74
|
+
end
|
75
|
+
|
76
|
+
# get the full URL of the worker api server
|
77
|
+
def self.api_url()
|
78
|
+
"#{BlackStack::Pampa::api_protocol}://#{BlackStack::Pampa::api_domain}:#{BlackStack::Pampa::api_port}"
|
79
|
+
end
|
80
|
+
|
81
|
+
#
|
82
|
+
def self.set_api_key(s)
|
83
|
+
@@api_key = s
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
def self.set_api_url(h)
|
88
|
+
@@api_key = h[:api_key]
|
89
|
+
@@api_protocol = h[:api_protocol]
|
90
|
+
@@api_domain = h[:api_domain]
|
91
|
+
@@api_port = h[:api_port]
|
92
|
+
end
|
93
|
+
|
94
|
+
# path where you will store the data of each client
|
95
|
+
@@storage_folder = nil
|
96
|
+
@@storage_sub_folders = []
|
97
|
+
|
98
|
+
#
|
99
|
+
def self.storage_folder()
|
100
|
+
@@storage_folder
|
101
|
+
end
|
102
|
+
def self.storage_sub_folders()
|
103
|
+
@@storage_sub_folders
|
104
|
+
end
|
105
|
+
|
106
|
+
#
|
107
|
+
def self.set_storage_folder(path)
|
108
|
+
@@storage_folder = path
|
109
|
+
end
|
110
|
+
def self.set_storage_sub_folders(a)
|
111
|
+
@@storage_sub_folders = a
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
#
|
116
|
+
# default timezome for any new user
|
117
|
+
#
|
118
|
+
#
|
119
|
+
@@id_timezone_default = nil
|
120
|
+
|
121
|
+
#
|
122
|
+
def self.id_timezone_default()
|
123
|
+
@@id_timezone_default
|
124
|
+
end
|
125
|
+
|
126
|
+
#
|
127
|
+
def self.set_id_timezone_default(id)
|
128
|
+
@@id_timezone_default = id
|
129
|
+
end
|
130
|
+
|
131
|
+
# Array of external IP addresses of the servers where farms are running.
|
132
|
+
# If you have many servers running behind a NAT server, only add the IP
|
133
|
+
# of the external gateway here. This array is used to know if a worker
|
134
|
+
# is running inside your farm datacenter, or it is running in the computer
|
135
|
+
# of someone else.
|
136
|
+
@@farm_external_ip_addresses = []
|
137
|
+
|
138
|
+
#
|
139
|
+
def self.farm_external_ip_addresses()
|
140
|
+
@@farm_external_ip_addresses
|
141
|
+
end
|
142
|
+
|
143
|
+
#
|
144
|
+
def self.set_farm_external_ip_addresses(a)
|
145
|
+
@@farm_external_ip_addresses = a
|
146
|
+
end
|
147
|
+
|
148
|
+
#
|
149
|
+
def self.get_guid
|
150
|
+
res = BlackStack::Netting::call_post(
|
151
|
+
"#{self.api_url}/api1.4/get_guid.json",
|
152
|
+
{'api_key' => @@api_key}
|
153
|
+
)
|
154
|
+
parsed = JSON.parse(res.body)
|
155
|
+
parsed['value']
|
156
|
+
end
|
157
|
+
|
158
|
+
# Central database connection parameters
|
159
|
+
@@db_url = nil
|
160
|
+
@@db_port = nil
|
161
|
+
@@db_name = nil
|
162
|
+
@@db_user = nil
|
163
|
+
@@db_password = nil
|
164
|
+
|
165
|
+
#
|
166
|
+
def self.db_url
|
167
|
+
@@db_url
|
168
|
+
end
|
169
|
+
|
170
|
+
#
|
171
|
+
def self.db_port
|
172
|
+
@@db_port
|
173
|
+
end
|
174
|
+
|
175
|
+
#
|
176
|
+
def self.db_name
|
177
|
+
@@db_name
|
178
|
+
end
|
179
|
+
|
180
|
+
#
|
181
|
+
def self.db_user
|
182
|
+
@@db_user
|
183
|
+
end
|
184
|
+
|
185
|
+
#
|
186
|
+
def self.db_password
|
187
|
+
@@db_password
|
188
|
+
end
|
189
|
+
|
190
|
+
# Set connection params to the central database
|
191
|
+
def self.set_db_params(h)
|
192
|
+
@@db_url = h[:db_url]
|
193
|
+
@@db_port = h[:db_port]
|
194
|
+
@@db_name = h[:db_name]
|
195
|
+
@@db_user = h[:db_user]
|
196
|
+
@@db_password = h[:db_password]
|
197
|
+
end
|
198
|
+
|
199
|
+
# TODO: doc me!
|
200
|
+
def self.connection_descriptor()
|
201
|
+
ret = nil
|
202
|
+
|
203
|
+
# validar que el formato no sea nulo
|
204
|
+
if (self.division_name.to_s.length == 0)
|
205
|
+
raise "Division name expected."
|
206
|
+
end
|
207
|
+
|
208
|
+
if (self.division_name == "local")
|
209
|
+
ret = {
|
210
|
+
:adapter => 'tinytds',
|
211
|
+
:dataserver => BlackStack::Pampa::db_url, # IP or hostname
|
212
|
+
:port => BlackStack::Pampa::db_port, # Required when using other that 1433 (default)
|
213
|
+
:database => BlackStack::Pampa::db_name,
|
214
|
+
:user => BlackStack::Pampa::db_user,
|
215
|
+
:password => BlackStack::Pampa::db_password
|
216
|
+
}
|
217
|
+
else
|
218
|
+
url = "#{BlackStack::Pampa::api_url}/api1.2/division/get.json"
|
219
|
+
res = BlackStack::Netting::call_post(url, {
|
220
|
+
'api_key' => BlackStack::Pampa::api_key,
|
221
|
+
'dname' => "#{self.division_name}",
|
222
|
+
})
|
223
|
+
parsed = JSON.parse(res.body)
|
224
|
+
|
225
|
+
if (parsed["status"] != BlackStack::Netting::SUCCESS)
|
226
|
+
raise "Error getting connection string: #{parsed["status"]}"
|
227
|
+
else
|
228
|
+
wid = parsed["value"]
|
229
|
+
|
230
|
+
ret = {
|
231
|
+
:adapter => 'tinytds',
|
232
|
+
:dataserver => parsed["db_url"], # IP or hostname
|
233
|
+
:port => parsed["db_port"], # only required if port is different than 1433
|
234
|
+
:database => parsed["db_name"],
|
235
|
+
:user => parsed["db_user"],
|
236
|
+
:password => parsed["db_password"]
|
237
|
+
}
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
ret
|
242
|
+
end # connectionDescriptor
|
243
|
+
|
244
|
+
#
|
245
|
+
def self.db_connection()
|
246
|
+
Sequel.connect(BlackStack::Pampa::connection_descriptor)
|
247
|
+
end
|
248
|
+
|
249
|
+
#
|
250
|
+
def self.require_db_classes()
|
251
|
+
# You have to load all the Sinatra classes after connect the database.
|
252
|
+
require_relative '../lib/pampa-local.rb'
|
253
|
+
end
|
254
|
+
|
255
|
+
end # module Pampa
|
256
|
+
|
257
|
+
end # module BlackStack
|
data/lib/params.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
module BlackStack
|
2
|
+
|
3
|
+
class Params < Sequel::Model(:params)
|
4
|
+
Params.dataset = Params.dataset.disable_insert_output
|
5
|
+
|
6
|
+
def self.getValue(s)
|
7
|
+
param = Params.where(:name=>s).first
|
8
|
+
if (param == nil)
|
9
|
+
raise "Unknown parameter name (#{s.to_s})"
|
10
|
+
else
|
11
|
+
if param.type == PARAM_TYPE_VARCHAR
|
12
|
+
return param.value_varchar.to_s
|
13
|
+
elsif param.type == PARAM_TYPE_NUMERIC
|
14
|
+
return param.value_numeric.to_i
|
15
|
+
elsif param.type == PARAM_TYPE_DATETIME
|
16
|
+
return param.value_datetime
|
17
|
+
elsif param.type == PARAM_TYPE_BIT
|
18
|
+
return param.value_bit
|
19
|
+
else
|
20
|
+
raise "Unknown parameter type (#{param.type.to_s})."
|
21
|
+
end # if param.type
|
22
|
+
end # if (param == nil)
|
23
|
+
end # def self.getValue
|
24
|
+
|
25
|
+
def self.setValue(s, v) # TODO: Testear este metodo
|
26
|
+
param = Params.where(:name=>s).first
|
27
|
+
if (param == nil)
|
28
|
+
raise "Unknown parameter name (#{s.to_s})"
|
29
|
+
else
|
30
|
+
if param.type == PARAM_TYPE_VARCHAR
|
31
|
+
param.value_varchar = v
|
32
|
+
param.save()
|
33
|
+
elsif param.type == PARAM_TYPE_NUMERIC
|
34
|
+
param.value_numeric = v
|
35
|
+
param.save()
|
36
|
+
elsif param.type == PARAM_TYPE_DATETIME
|
37
|
+
param.value_datetime = v
|
38
|
+
param.save()
|
39
|
+
elsif param.type == PARAM_TYPE_BIT
|
40
|
+
param.value_bit = v
|
41
|
+
param.save()
|
42
|
+
else
|
43
|
+
raise "Unknown parameter type (#{param.type.to_s})."
|
44
|
+
end # if param.type
|
45
|
+
end # if (param == nil)
|
46
|
+
end # def self.getValue
|
47
|
+
|
48
|
+
end # class
|
49
|
+
|
50
|
+
end # module BlackStack
|