pampa_workers 0.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/basedivision.rb +5 -0
- data/lib/baseworker.rb +5 -0
- data/lib/client.rb +245 -0
- data/lib/division.rb +46 -0
- data/lib/login.rb +7 -0
- data/lib/mybotprocess.rb +348 -0
- data/lib/mychildprocess.rb +9 -0
- data/lib/mycrawlprocess.rb +49 -0
- data/lib/mylocalprocess.rb +164 -0
- data/lib/myparentprocess.rb +141 -0
- data/lib/myprocess.rb +264 -0
- data/lib/myremoteprocess.rb +128 -0
- data/lib/pampa-local.rb +41 -0
- data/lib/pampa.rb +261 -0
- data/lib/params.rb +50 -0
- data/lib/remotedivision.rb +8 -0
- data/lib/remoteworker.rb +8 -0
- data/lib/role.rb +8 -0
- data/lib/timezone.rb +151 -0
- data/lib/user.rb +25 -0
- data/lib/userdivision.rb +5 -0
- data/lib/userrole.rb +8 -0
- data/lib/worker.rb +234 -0
- metadata +229 -0
data/lib/worker.rb
ADDED
@@ -0,0 +1,234 @@
|
|
1
|
+
module BlackStack
|
2
|
+
|
3
|
+
#
|
4
|
+
class WorkerJob < Sequel::Model(:workerjob)
|
5
|
+
|
6
|
+
end
|
7
|
+
|
8
|
+
#
|
9
|
+
class Worker < Sequel::Model(:worker)
|
10
|
+
include BlackStack::BaseWorker
|
11
|
+
BlackStack::Worker.dataset = BlackStack::Worker.dataset.disable_insert_output
|
12
|
+
many_to_one :division, :class=>:'BlackStack::Division', :key=>:id_division
|
13
|
+
many_to_one :user, :class=>:'BlackStack::User', :key=>:id_user
|
14
|
+
=begin
|
15
|
+
# deprecated
|
16
|
+
# Actualiza la la lista de workers que estan asignados esta division
|
17
|
+
def self.updateAllFromCentral()
|
18
|
+
uri = URI("#{WORKER_API_SERVER_URL}/api1.3/pampa/get_all.json")
|
19
|
+
res = Net::HTTP.post_form(uri, {:api_key => BlackStack::Pampa::api_key,})
|
20
|
+
parsed = JSON.parse(res.body)
|
21
|
+
if (parsed['status'] != "success")
|
22
|
+
raise parsed['status'].to_s
|
23
|
+
else
|
24
|
+
parsed['workers'].each { |worker|
|
25
|
+
if ( worker['division_name']!=DATABASE )
|
26
|
+
q = "UPDATE worker SET active=0, division_name='#{worker['division_name']}' WHERE name='#{worker['name']}'"
|
27
|
+
DB.execute(q)
|
28
|
+
else # worker['division_name']==DIVISION_NAME
|
29
|
+
|
30
|
+
worker = BlackStack::Worker.where(:name=>worker['name']).first
|
31
|
+
if (worker==nil)
|
32
|
+
worker = BlackStack::Worker.new()
|
33
|
+
worker.id = worker['id']
|
34
|
+
worker.name = worker['name'].to_s
|
35
|
+
worker.last_ping_time = now() # esta fecha es actualiada por el mismo worker, para indicar que esta vivo y trabajando
|
36
|
+
worker.id_division = worker['id_division']
|
37
|
+
worker.process = worker['assigned_process']
|
38
|
+
worker.assigned_process = worker['assigned_process']
|
39
|
+
worker.id_object = worker['id_object']
|
40
|
+
worker.division_name = worker['division_name']
|
41
|
+
worker.save()
|
42
|
+
else
|
43
|
+
#puts "update" ?
|
44
|
+
end
|
45
|
+
|
46
|
+
DB.execute("UPDATE worker SET active=1 WHERE name='#{worker['name'].to_s}'")
|
47
|
+
|
48
|
+
if (worker['id_division'] != nil)
|
49
|
+
DB.execute("UPDATE worker SET id_division='#{worker['id_division'].to_s}' WHERE name='#{worker['name'].to_s}'")
|
50
|
+
end
|
51
|
+
|
52
|
+
if (worker['assigned_process'] != nil)
|
53
|
+
DB.execute("UPDATE worker SET process='#{worker['assigned_process'].to_s}', assigned_process='#{worker['assigned_process'].to_s}' WHERE name='#{worker['name'].to_s}'")
|
54
|
+
end
|
55
|
+
|
56
|
+
if (worker['id_object'] != nil)
|
57
|
+
DB.execute("UPDATE worker SET id_object='#{worker['id_object'].to_s}' WHERE name='#{worker['name'].to_s}'")
|
58
|
+
end
|
59
|
+
|
60
|
+
if (worker['division_name'] != nil)
|
61
|
+
DB.execute("UPDATE worker SET division_name='#{worker['division_name'].to_s}' WHERE name='#{worker['name'].to_s}'")
|
62
|
+
end
|
63
|
+
|
64
|
+
if (worker['type']==nil || worker['type'].to_i==MyProcess::TYPE_LOCAL)
|
65
|
+
DB.execute("UPDATE worker SET type=#{MyProcess::TYPE_LOCAL.to_s} WHERE name='#{worker['name'].to_s}'")
|
66
|
+
else
|
67
|
+
DB.execute("UPDATE worker SET type=#{MyProcess::TYPE_REMOTE.to_s} WHERE name='#{worker['name'].to_s}'")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# release resources
|
72
|
+
DB.disconnect
|
73
|
+
GC.start
|
74
|
+
}
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
# actualiza el rol y objeto asignado a este worker
|
80
|
+
def updateFromCentral()
|
81
|
+
uri = URI("#{WORKER_API_SERVER_URL}/api1.3/pampa/get.json")
|
82
|
+
res = Net::HTTP.post_form(uri, {'api_key' => BlackStack::Pampa::api_key, 'name' => self.name})
|
83
|
+
parsed = JSON.parse(res.body)
|
84
|
+
if (parsed['status'] != "success")
|
85
|
+
raise parsed['status'].to_s
|
86
|
+
else
|
87
|
+
# map response
|
88
|
+
self.id_division = parsed['id_division']
|
89
|
+
self.assigned_process = parsed['assigned_process']
|
90
|
+
self.id_object = parsed['id_object']
|
91
|
+
self.division_name = parsed['division_name']
|
92
|
+
self.active = true
|
93
|
+
|
94
|
+
if (parsed['id_object'].to_s.size>0)
|
95
|
+
aux_id_object = "'#{parsed['id_object']}'"
|
96
|
+
else
|
97
|
+
aux_id_object = "NULL"
|
98
|
+
end
|
99
|
+
|
100
|
+
# NOTA: DEBO HACER EL UPDATE POR FUERA DE SQUEL, DEBIDO AL BUG DE MAPEO DE SEQUEL
|
101
|
+
q =
|
102
|
+
"UPDATE worker SET " +
|
103
|
+
"active=1, id_division='#{parsed['id_division']}', assigned_process='#{parsed['assigned_process'].to_s.gsub("'","''")}', id_object=#{aux_id_object}, division_name='#{parsed['division_name'].to_s.gsub("'","''")}' " +
|
104
|
+
"WHERE id='#{self.id}'"
|
105
|
+
DB.execute(q)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
# actualiza el rol y objeto asignado a cada worker asignado a esta division
|
110
|
+
def self.updateActivesFromCentral()
|
111
|
+
BlackStack::Worker.getActives().each { |worker|
|
112
|
+
worker.updateFromCentral()
|
113
|
+
}
|
114
|
+
end
|
115
|
+
=end
|
116
|
+
|
117
|
+
#
|
118
|
+
def factory(name, params)
|
119
|
+
w = BlackStack::Worker.where(:name=>name).first
|
120
|
+
if (w==nil)
|
121
|
+
w = BlackStack::Worker.new
|
122
|
+
end
|
123
|
+
w.id = parsed['id']
|
124
|
+
w.name = name
|
125
|
+
w.assigned_process = parsed['assigned_process']
|
126
|
+
w.id_object = parsed['id_object']
|
127
|
+
w.id_division = parsed['id_division']
|
128
|
+
w.division_name = parsed['division_name']
|
129
|
+
w.ws_url = parsed['ws_url']
|
130
|
+
w.ws_port = parsed['ws_port']
|
131
|
+
w.save
|
132
|
+
end
|
133
|
+
|
134
|
+
# Retorna true si este worker esta corriendo en nuestros propios servidores,
|
135
|
+
# Retorna false si este worker esta correiendo en otro host, asumiendo que es el host del cliente.
|
136
|
+
# Comparando la pulic_ip_address del worer con la lista en BlackStack::Pampa::set_farm_external_ip_addresses.
|
137
|
+
def hosted?
|
138
|
+
BlackStack::Pampa::set_farm_external_ip_addresses.inlude?(self.public_ip_address)
|
139
|
+
end # hosted?
|
140
|
+
|
141
|
+
# Si es un worker hosteado en nuestos servidores (ver metodo hosted?),
|
142
|
+
# => retorna la cantidad de dias que fa
|
143
|
+
def expirationDesc
|
144
|
+
s = "(unknown)"
|
145
|
+
if self.hosted?
|
146
|
+
if !self.expiration_time.nil?
|
147
|
+
s = DB["SELECT DATEDIFF(mi, GETDATE(), w.expiration_time) AS n FROM worker w WHERE w.id='#{self.id}'"].first[:n].to_i.to_time_spent
|
148
|
+
end
|
149
|
+
else # no hosted
|
150
|
+
s = "(self-hosted)"
|
151
|
+
end
|
152
|
+
s
|
153
|
+
end
|
154
|
+
|
155
|
+
# Retorna la cantidad de minutos desde que este worker envio una senial de vida.
|
156
|
+
# Este metodo se usa para saber si un worker esta activo o no.
|
157
|
+
def last_ping_minutes()
|
158
|
+
q = "SELECT DATEDIFF(mi, p.last_ping_time, getdate()) AS minutes FROM worker p WHERE p.id='#{self.id}'"
|
159
|
+
return DB[q].first[:minutes].to_i
|
160
|
+
end
|
161
|
+
|
162
|
+
# returns true if this worker had got a ping within the last 5 minutes
|
163
|
+
def active?
|
164
|
+
self.last_ping_minutes < BlackStack::BaseWorker::KEEP_ACTIVE_MINUTES
|
165
|
+
end
|
166
|
+
|
167
|
+
# escribe en el archivo de log de este worker
|
168
|
+
def log(s, level=1, is_error=false)
|
169
|
+
logw(s, self.process, self.id, level, is_error)
|
170
|
+
end
|
171
|
+
|
172
|
+
# envia una senial de vida a la division
|
173
|
+
# TODO: guardar fecha-hora del ultimo ping en un atributo privado, y evitar el acceso escesivo a la base de datos
|
174
|
+
def ping()
|
175
|
+
DB.execute("UPDATE worker SET last_ping_time=GETDATE() WHERE id='#{self.id}'")
|
176
|
+
end
|
177
|
+
|
178
|
+
# DEPRECATED
|
179
|
+
def self.getActivesCount(processName)
|
180
|
+
raise "Method needs some code inside."
|
181
|
+
end
|
182
|
+
|
183
|
+
# obtiene array de workers actives, filtrados por proceso y por tipo de worker.
|
184
|
+
def self.getActives(assigned_process_name=nil, worker_name_filter=nil)
|
185
|
+
a = Array.new
|
186
|
+
q = ""
|
187
|
+
if (assigned_process_name!=nil)
|
188
|
+
q =
|
189
|
+
"SELECT p.id AS [id] " +
|
190
|
+
"FROM worker p WITH (NOLOCK INDEX(IX_peer__process__last_ping_time)) " +
|
191
|
+
"WHERE last_ping_time>DATEADD(mi,-5,GETDATE()) " +
|
192
|
+
"AND ISNULL(active,0)=1 " + # active indica si este worker fue asignado a esta division en la central
|
193
|
+
"AND assigned_process='#{assigned_process_name}' "
|
194
|
+
|
195
|
+
if worker_name_filter != nil
|
196
|
+
q = q +
|
197
|
+
"AND p.name LIKE '%#{worker_name_filter.to_s}%' "
|
198
|
+
end
|
199
|
+
|
200
|
+
q = q +
|
201
|
+
"ORDER BY p.name "
|
202
|
+
DB[q].all do |row|
|
203
|
+
a << BlackStack::Worker.where(:id=>row[:id]).first
|
204
|
+
end
|
205
|
+
else
|
206
|
+
q =
|
207
|
+
"SELECT p.id AS [id] " +
|
208
|
+
"FROM worker p WITH (NOLOCK INDEX(IX_peer__process__last_ping_time)) " +
|
209
|
+
"WHERE last_ping_time>DATEADD(mi,-5,GETDATE()) " +
|
210
|
+
"AND ISNULL(active,0)=1 "
|
211
|
+
|
212
|
+
if worker_name_filter != nil
|
213
|
+
q = q +
|
214
|
+
"AND p.name LIKE '%#{worker_name_filter.to_s}%' "
|
215
|
+
end
|
216
|
+
|
217
|
+
q = q +
|
218
|
+
"ORDER BY p.name "
|
219
|
+
DB[q].all do |row|
|
220
|
+
a << BlackStack::Worker.where(:id=>row[:id]).first
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
return a
|
225
|
+
end
|
226
|
+
|
227
|
+
# obtiene cantidad de registros en cola para incrawl.lnsearchvariation
|
228
|
+
def getPendingLnSearchVariationBlockInCrawlCount()
|
229
|
+
return DB.from(:lnsearchvariationblock).where(:incrawl_reservation_id=>self.id, :incrawl_start_time=>nil).count
|
230
|
+
end
|
231
|
+
|
232
|
+
end # class Worker
|
233
|
+
|
234
|
+
end # module BlackStack
|
metadata
ADDED
@@ -0,0 +1,229 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pampa_workers
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.38
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Leandro Daniel Sardi
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-12-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: websocket
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.2.8
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 1.2.8
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - "~>"
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.2.8
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.2.8
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: json
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 1.8.1
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 1.8.1
|
43
|
+
type: :runtime
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: 1.8.1
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 1.8.1
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: tiny_tds
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - "~>"
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: 1.0.5
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 1.0.5
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.0.5
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: 1.0.5
|
73
|
+
- !ruby/object:Gem::Dependency
|
74
|
+
name: sequel
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - "~>"
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: 4.28.0
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 4.28.0
|
83
|
+
type: :runtime
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 4.28.0
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: 4.28.0
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: blackstack_commons
|
95
|
+
requirement: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - "~>"
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: 0.0.20
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: 0.0.20
|
103
|
+
type: :runtime
|
104
|
+
prerelease: false
|
105
|
+
version_requirements: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - "~>"
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 0.0.20
|
110
|
+
- - ">="
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: 0.0.20
|
113
|
+
- !ruby/object:Gem::Dependency
|
114
|
+
name: simple_cloud_logging
|
115
|
+
requirement: !ruby/object:Gem::Requirement
|
116
|
+
requirements:
|
117
|
+
- - "~>"
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: 1.1.16
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: 1.1.16
|
123
|
+
type: :runtime
|
124
|
+
prerelease: false
|
125
|
+
version_requirements: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - "~>"
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: 1.1.16
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: 1.1.16
|
133
|
+
- !ruby/object:Gem::Dependency
|
134
|
+
name: simple_command_line_parser
|
135
|
+
requirement: !ruby/object:Gem::Requirement
|
136
|
+
requirements:
|
137
|
+
- - "~>"
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
version: 1.1.1
|
140
|
+
- - ">="
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: 1.1.1
|
143
|
+
type: :runtime
|
144
|
+
prerelease: false
|
145
|
+
version_requirements: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - "~>"
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 1.1.1
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: 1.1.1
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: simple_host_monitoring
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: 0.0.11
|
160
|
+
- - ">="
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: 0.0.11
|
163
|
+
type: :runtime
|
164
|
+
prerelease: false
|
165
|
+
version_requirements: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - "~>"
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: 0.0.11
|
170
|
+
- - ">="
|
171
|
+
- !ruby/object:Gem::Version
|
172
|
+
version: 0.0.11
|
173
|
+
description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/pampa.'
|
174
|
+
email: leandro.sardi@expandedventure.com
|
175
|
+
executables: []
|
176
|
+
extensions: []
|
177
|
+
extra_rdoc_files: []
|
178
|
+
files:
|
179
|
+
- lib/basedivision.rb
|
180
|
+
- lib/baseworker.rb
|
181
|
+
- lib/client.rb
|
182
|
+
- lib/division.rb
|
183
|
+
- lib/login.rb
|
184
|
+
- lib/mybotprocess.rb
|
185
|
+
- lib/mychildprocess.rb
|
186
|
+
- lib/mycrawlprocess.rb
|
187
|
+
- lib/mylocalprocess.rb
|
188
|
+
- lib/myparentprocess.rb
|
189
|
+
- lib/myprocess.rb
|
190
|
+
- lib/myremoteprocess.rb
|
191
|
+
- lib/pampa-local.rb
|
192
|
+
- lib/pampa.rb
|
193
|
+
- lib/params.rb
|
194
|
+
- lib/remotedivision.rb
|
195
|
+
- lib/remoteworker.rb
|
196
|
+
- lib/role.rb
|
197
|
+
- lib/timezone.rb
|
198
|
+
- lib/user.rb
|
199
|
+
- lib/userdivision.rb
|
200
|
+
- lib/userrole.rb
|
201
|
+
- lib/worker.rb
|
202
|
+
homepage: https://rubygems.org/gems/pampa
|
203
|
+
licenses:
|
204
|
+
- MIT
|
205
|
+
metadata: {}
|
206
|
+
post_install_message:
|
207
|
+
rdoc_options: []
|
208
|
+
require_paths:
|
209
|
+
- lib
|
210
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
211
|
+
requirements:
|
212
|
+
- - ">="
|
213
|
+
- !ruby/object:Gem::Version
|
214
|
+
version: '0'
|
215
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
216
|
+
requirements:
|
217
|
+
- - ">="
|
218
|
+
- !ruby/object:Gem::Version
|
219
|
+
version: '0'
|
220
|
+
requirements: []
|
221
|
+
rubyforge_project:
|
222
|
+
rubygems_version: 2.4.5.1
|
223
|
+
signing_key:
|
224
|
+
specification_version: 4
|
225
|
+
summary: THIS GEM IS STILL IN DEVELOPMENT STAGE. Ruby library for distributing computing,
|
226
|
+
supporting dynamic reconfiguration, distribution of the computation jobs, error
|
227
|
+
handling, job-retry and fault tolerance, fast (non-direct) communication to ensure
|
228
|
+
real-time capabilities.
|
229
|
+
test_files: []
|