pampa_workers 0.0.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. checksums.yaml +7 -0
  2. data/lib/basedivision.rb +5 -0
  3. data/lib/baseworker.rb +5 -0
  4. data/lib/client.rb +245 -0
  5. data/lib/division.rb +46 -0
  6. data/lib/login.rb +7 -0
  7. data/lib/mybotprocess.rb +348 -0
  8. data/lib/mychildprocess.rb +9 -0
  9. data/lib/mycrawlprocess.rb +49 -0
  10. data/lib/mylocalprocess.rb +164 -0
  11. data/lib/myparentprocess.rb +141 -0
  12. data/lib/myprocess.rb +264 -0
  13. data/lib/myremoteprocess.rb +128 -0
  14. data/lib/pampa-local.rb +41 -0
  15. data/lib/pampa.rb +261 -0
  16. data/lib/params.rb +50 -0
  17. data/lib/remotedivision.rb +8 -0
  18. data/lib/remoteworker.rb +8 -0
  19. data/lib/role.rb +8 -0
  20. data/lib/timezone.rb +151 -0
  21. data/lib/user.rb +25 -0
  22. data/lib/userdivision.rb +5 -0
  23. data/lib/userrole.rb +8 -0
  24. data/lib/worker.rb +234 -0
  25. metadata +229 -0
data/lib/worker.rb ADDED
@@ -0,0 +1,234 @@
1
+ module BlackStack
2
+
3
+ #
4
+ class WorkerJob < Sequel::Model(:workerjob)
5
+
6
+ end
7
+
8
+ #
9
+ class Worker < Sequel::Model(:worker)
10
+ include BlackStack::BaseWorker
11
+ BlackStack::Worker.dataset = BlackStack::Worker.dataset.disable_insert_output
12
+ many_to_one :division, :class=>:'BlackStack::Division', :key=>:id_division
13
+ many_to_one :user, :class=>:'BlackStack::User', :key=>:id_user
14
+ =begin
15
+ # deprecated
16
+ # Actualiza la la lista de workers que estan asignados esta division
17
+ def self.updateAllFromCentral()
18
+ uri = URI("#{WORKER_API_SERVER_URL}/api1.3/pampa/get_all.json")
19
+ res = Net::HTTP.post_form(uri, {:api_key => BlackStack::Pampa::api_key,})
20
+ parsed = JSON.parse(res.body)
21
+ if (parsed['status'] != "success")
22
+ raise parsed['status'].to_s
23
+ else
24
+ parsed['workers'].each { |worker|
25
+ if ( worker['division_name']!=DATABASE )
26
+ q = "UPDATE worker SET active=0, division_name='#{worker['division_name']}' WHERE name='#{worker['name']}'"
27
+ DB.execute(q)
28
+ else # worker['division_name']==DIVISION_NAME
29
+
30
+ worker = BlackStack::Worker.where(:name=>worker['name']).first
31
+ if (worker==nil)
32
+ worker = BlackStack::Worker.new()
33
+ worker.id = worker['id']
34
+ worker.name = worker['name'].to_s
35
+ worker.last_ping_time = now() # esta fecha es actualiada por el mismo worker, para indicar que esta vivo y trabajando
36
+ worker.id_division = worker['id_division']
37
+ worker.process = worker['assigned_process']
38
+ worker.assigned_process = worker['assigned_process']
39
+ worker.id_object = worker['id_object']
40
+ worker.division_name = worker['division_name']
41
+ worker.save()
42
+ else
43
+ #puts "update" ?
44
+ end
45
+
46
+ DB.execute("UPDATE worker SET active=1 WHERE name='#{worker['name'].to_s}'")
47
+
48
+ if (worker['id_division'] != nil)
49
+ DB.execute("UPDATE worker SET id_division='#{worker['id_division'].to_s}' WHERE name='#{worker['name'].to_s}'")
50
+ end
51
+
52
+ if (worker['assigned_process'] != nil)
53
+ DB.execute("UPDATE worker SET process='#{worker['assigned_process'].to_s}', assigned_process='#{worker['assigned_process'].to_s}' WHERE name='#{worker['name'].to_s}'")
54
+ end
55
+
56
+ if (worker['id_object'] != nil)
57
+ DB.execute("UPDATE worker SET id_object='#{worker['id_object'].to_s}' WHERE name='#{worker['name'].to_s}'")
58
+ end
59
+
60
+ if (worker['division_name'] != nil)
61
+ DB.execute("UPDATE worker SET division_name='#{worker['division_name'].to_s}' WHERE name='#{worker['name'].to_s}'")
62
+ end
63
+
64
+ if (worker['type']==nil || worker['type'].to_i==MyProcess::TYPE_LOCAL)
65
+ DB.execute("UPDATE worker SET type=#{MyProcess::TYPE_LOCAL.to_s} WHERE name='#{worker['name'].to_s}'")
66
+ else
67
+ DB.execute("UPDATE worker SET type=#{MyProcess::TYPE_REMOTE.to_s} WHERE name='#{worker['name'].to_s}'")
68
+ end
69
+ end
70
+
71
+ # release resources
72
+ DB.disconnect
73
+ GC.start
74
+ }
75
+
76
+ end
77
+ end
78
+
79
+ # actualiza el rol y objeto asignado a este worker
80
+ def updateFromCentral()
81
+ uri = URI("#{WORKER_API_SERVER_URL}/api1.3/pampa/get.json")
82
+ res = Net::HTTP.post_form(uri, {'api_key' => BlackStack::Pampa::api_key, 'name' => self.name})
83
+ parsed = JSON.parse(res.body)
84
+ if (parsed['status'] != "success")
85
+ raise parsed['status'].to_s
86
+ else
87
+ # map response
88
+ self.id_division = parsed['id_division']
89
+ self.assigned_process = parsed['assigned_process']
90
+ self.id_object = parsed['id_object']
91
+ self.division_name = parsed['division_name']
92
+ self.active = true
93
+
94
+ if (parsed['id_object'].to_s.size>0)
95
+ aux_id_object = "'#{parsed['id_object']}'"
96
+ else
97
+ aux_id_object = "NULL"
98
+ end
99
+
100
+ # NOTA: DEBO HACER EL UPDATE POR FUERA DE SQUEL, DEBIDO AL BUG DE MAPEO DE SEQUEL
101
+ q =
102
+ "UPDATE worker SET " +
103
+ "active=1, id_division='#{parsed['id_division']}', assigned_process='#{parsed['assigned_process'].to_s.gsub("'","''")}', id_object=#{aux_id_object}, division_name='#{parsed['division_name'].to_s.gsub("'","''")}' " +
104
+ "WHERE id='#{self.id}'"
105
+ DB.execute(q)
106
+ end
107
+ end
108
+
109
+ # actualiza el rol y objeto asignado a cada worker asignado a esta division
110
+ def self.updateActivesFromCentral()
111
+ BlackStack::Worker.getActives().each { |worker|
112
+ worker.updateFromCentral()
113
+ }
114
+ end
115
+ =end
116
+
117
+ #
118
+ def factory(name, params)
119
+ w = BlackStack::Worker.where(:name=>name).first
120
+ if (w==nil)
121
+ w = BlackStack::Worker.new
122
+ end
123
+ w.id = parsed['id']
124
+ w.name = name
125
+ w.assigned_process = parsed['assigned_process']
126
+ w.id_object = parsed['id_object']
127
+ w.id_division = parsed['id_division']
128
+ w.division_name = parsed['division_name']
129
+ w.ws_url = parsed['ws_url']
130
+ w.ws_port = parsed['ws_port']
131
+ w.save
132
+ end
133
+
134
+ # Retorna true si este worker esta corriendo en nuestros propios servidores,
135
+ # Retorna false si este worker esta correiendo en otro host, asumiendo que es el host del cliente.
136
+ # Comparando la pulic_ip_address del worer con la lista en BlackStack::Pampa::set_farm_external_ip_addresses.
137
+ def hosted?
138
+ BlackStack::Pampa::set_farm_external_ip_addresses.inlude?(self.public_ip_address)
139
+ end # hosted?
140
+
141
+ # Si es un worker hosteado en nuestos servidores (ver metodo hosted?),
142
+ # => retorna la cantidad de dias que fa
143
+ def expirationDesc
144
+ s = "(unknown)"
145
+ if self.hosted?
146
+ if !self.expiration_time.nil?
147
+ s = DB["SELECT DATEDIFF(mi, GETDATE(), w.expiration_time) AS n FROM worker w WHERE w.id='#{self.id}'"].first[:n].to_i.to_time_spent
148
+ end
149
+ else # no hosted
150
+ s = "(self-hosted)"
151
+ end
152
+ s
153
+ end
154
+
155
+ # Retorna la cantidad de minutos desde que este worker envio una senial de vida.
156
+ # Este metodo se usa para saber si un worker esta activo o no.
157
+ def last_ping_minutes()
158
+ q = "SELECT DATEDIFF(mi, p.last_ping_time, getdate()) AS minutes FROM worker p WHERE p.id='#{self.id}'"
159
+ return DB[q].first[:minutes].to_i
160
+ end
161
+
162
+ # returns true if this worker had got a ping within the last 5 minutes
163
+ def active?
164
+ self.last_ping_minutes < BlackStack::BaseWorker::KEEP_ACTIVE_MINUTES
165
+ end
166
+
167
+ # escribe en el archivo de log de este worker
168
+ def log(s, level=1, is_error=false)
169
+ logw(s, self.process, self.id, level, is_error)
170
+ end
171
+
172
+ # envia una senial de vida a la division
173
+ # TODO: guardar fecha-hora del ultimo ping en un atributo privado, y evitar el acceso escesivo a la base de datos
174
+ def ping()
175
+ DB.execute("UPDATE worker SET last_ping_time=GETDATE() WHERE id='#{self.id}'")
176
+ end
177
+
178
+ # DEPRECATED
179
+ def self.getActivesCount(processName)
180
+ raise "Method needs some code inside."
181
+ end
182
+
183
+ # obtiene array de workers actives, filtrados por proceso y por tipo de worker.
184
+ def self.getActives(assigned_process_name=nil, worker_name_filter=nil)
185
+ a = Array.new
186
+ q = ""
187
+ if (assigned_process_name!=nil)
188
+ q =
189
+ "SELECT p.id AS [id] " +
190
+ "FROM worker p WITH (NOLOCK INDEX(IX_peer__process__last_ping_time)) " +
191
+ "WHERE last_ping_time>DATEADD(mi,-5,GETDATE()) " +
192
+ "AND ISNULL(active,0)=1 " + # active indica si este worker fue asignado a esta division en la central
193
+ "AND assigned_process='#{assigned_process_name}' "
194
+
195
+ if worker_name_filter != nil
196
+ q = q +
197
+ "AND p.name LIKE '%#{worker_name_filter.to_s}%' "
198
+ end
199
+
200
+ q = q +
201
+ "ORDER BY p.name "
202
+ DB[q].all do |row|
203
+ a << BlackStack::Worker.where(:id=>row[:id]).first
204
+ end
205
+ else
206
+ q =
207
+ "SELECT p.id AS [id] " +
208
+ "FROM worker p WITH (NOLOCK INDEX(IX_peer__process__last_ping_time)) " +
209
+ "WHERE last_ping_time>DATEADD(mi,-5,GETDATE()) " +
210
+ "AND ISNULL(active,0)=1 "
211
+
212
+ if worker_name_filter != nil
213
+ q = q +
214
+ "AND p.name LIKE '%#{worker_name_filter.to_s}%' "
215
+ end
216
+
217
+ q = q +
218
+ "ORDER BY p.name "
219
+ DB[q].all do |row|
220
+ a << BlackStack::Worker.where(:id=>row[:id]).first
221
+ end
222
+ end
223
+
224
+ return a
225
+ end
226
+
227
+ # obtiene cantidad de registros en cola para incrawl.lnsearchvariation
228
+ def getPendingLnSearchVariationBlockInCrawlCount()
229
+ return DB.from(:lnsearchvariationblock).where(:incrawl_reservation_id=>self.id, :incrawl_start_time=>nil).count
230
+ end
231
+
232
+ end # class Worker
233
+
234
+ end # module BlackStack
metadata ADDED
@@ -0,0 +1,229 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: pampa_workers
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.38
5
+ platform: ruby
6
+ authors:
7
+ - Leandro Daniel Sardi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-12-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: websocket
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 1.2.8
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 1.2.8
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: 1.2.8
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 1.2.8
33
+ - !ruby/object:Gem::Dependency
34
+ name: json
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: 1.8.1
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 1.8.1
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: 1.8.1
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 1.8.1
53
+ - !ruby/object:Gem::Dependency
54
+ name: tiny_tds
55
+ requirement: !ruby/object:Gem::Requirement
56
+ requirements:
57
+ - - "~>"
58
+ - !ruby/object:Gem::Version
59
+ version: 1.0.5
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: 1.0.5
63
+ type: :runtime
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - "~>"
68
+ - !ruby/object:Gem::Version
69
+ version: 1.0.5
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: 1.0.5
73
+ - !ruby/object:Gem::Dependency
74
+ name: sequel
75
+ requirement: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: 4.28.0
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: 4.28.0
83
+ type: :runtime
84
+ prerelease: false
85
+ version_requirements: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: 4.28.0
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: 4.28.0
93
+ - !ruby/object:Gem::Dependency
94
+ name: blackstack_commons
95
+ requirement: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - "~>"
98
+ - !ruby/object:Gem::Version
99
+ version: 0.0.20
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: 0.0.20
103
+ type: :runtime
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: 0.0.20
110
+ - - ">="
111
+ - !ruby/object:Gem::Version
112
+ version: 0.0.20
113
+ - !ruby/object:Gem::Dependency
114
+ name: simple_cloud_logging
115
+ requirement: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - "~>"
118
+ - !ruby/object:Gem::Version
119
+ version: 1.1.16
120
+ - - ">="
121
+ - !ruby/object:Gem::Version
122
+ version: 1.1.16
123
+ type: :runtime
124
+ prerelease: false
125
+ version_requirements: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - "~>"
128
+ - !ruby/object:Gem::Version
129
+ version: 1.1.16
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: 1.1.16
133
+ - !ruby/object:Gem::Dependency
134
+ name: simple_command_line_parser
135
+ requirement: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - "~>"
138
+ - !ruby/object:Gem::Version
139
+ version: 1.1.1
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: 1.1.1
143
+ type: :runtime
144
+ prerelease: false
145
+ version_requirements: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - "~>"
148
+ - !ruby/object:Gem::Version
149
+ version: 1.1.1
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: 1.1.1
153
+ - !ruby/object:Gem::Dependency
154
+ name: simple_host_monitoring
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: 0.0.11
160
+ - - ">="
161
+ - !ruby/object:Gem::Version
162
+ version: 0.0.11
163
+ type: :runtime
164
+ prerelease: false
165
+ version_requirements: !ruby/object:Gem::Requirement
166
+ requirements:
167
+ - - "~>"
168
+ - !ruby/object:Gem::Version
169
+ version: 0.0.11
170
+ - - ">="
171
+ - !ruby/object:Gem::Version
172
+ version: 0.0.11
173
+ description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/pampa.'
174
+ email: leandro.sardi@expandedventure.com
175
+ executables: []
176
+ extensions: []
177
+ extra_rdoc_files: []
178
+ files:
179
+ - lib/basedivision.rb
180
+ - lib/baseworker.rb
181
+ - lib/client.rb
182
+ - lib/division.rb
183
+ - lib/login.rb
184
+ - lib/mybotprocess.rb
185
+ - lib/mychildprocess.rb
186
+ - lib/mycrawlprocess.rb
187
+ - lib/mylocalprocess.rb
188
+ - lib/myparentprocess.rb
189
+ - lib/myprocess.rb
190
+ - lib/myremoteprocess.rb
191
+ - lib/pampa-local.rb
192
+ - lib/pampa.rb
193
+ - lib/params.rb
194
+ - lib/remotedivision.rb
195
+ - lib/remoteworker.rb
196
+ - lib/role.rb
197
+ - lib/timezone.rb
198
+ - lib/user.rb
199
+ - lib/userdivision.rb
200
+ - lib/userrole.rb
201
+ - lib/worker.rb
202
+ homepage: https://rubygems.org/gems/pampa
203
+ licenses:
204
+ - MIT
205
+ metadata: {}
206
+ post_install_message:
207
+ rdoc_options: []
208
+ require_paths:
209
+ - lib
210
+ required_ruby_version: !ruby/object:Gem::Requirement
211
+ requirements:
212
+ - - ">="
213
+ - !ruby/object:Gem::Version
214
+ version: '0'
215
+ required_rubygems_version: !ruby/object:Gem::Requirement
216
+ requirements:
217
+ - - ">="
218
+ - !ruby/object:Gem::Version
219
+ version: '0'
220
+ requirements: []
221
+ rubyforge_project:
222
+ rubygems_version: 2.4.5.1
223
+ signing_key:
224
+ specification_version: 4
225
+ summary: THIS GEM IS STILL IN DEVELOPMENT STAGE. Ruby library for distributing computing,
226
+ supporting dynamic reconfiguration, distribution of the computation jobs, error
227
+ handling, job-retry and fault tolerance, fast (non-direct) communication to ensure
228
+ real-time capabilities.
229
+ test_files: []