pampa 2.0.14 → 2.0.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/pampa.rb +57 -49
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5f8fec0d0fcba61ab662150060b8ce14185a9d3c3d2d4a74a3c255da0ae3e8e6
|
4
|
+
data.tar.gz: d290f6eb64265a7a16451ea4b6e9be99f6e3ca4f86493703894b44fc8f57c275
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 49ccc2d9c1440d65ffd0b137425807ff0e34133363d7308f3a9d4fea050d24d078a319f608f0bb3a0f68622154603af4f3ec6f858ea83f743789d0d3b6cd1a2f
|
7
|
+
data.tar.gz: 49b313fa4e0ef847643e632249cf529dfdc0ecc357de0fac1f226694895f3959dc5e777002c9481ecff4946d6638e46184b64f919fae97f7ed5bb49785269bfe
|
data/lib/pampa.rb
CHANGED
@@ -23,6 +23,11 @@ module BlackStack
|
|
23
23
|
# Connection string to the database. Example: mysql2://user:password@localhost:3306/database
|
24
24
|
@@connection_string = nil
|
25
25
|
|
26
|
+
def self.now()
|
27
|
+
tz = 'America/Argentina/Buenos_Aires' #DB["SELECT current_setting('TIMEZONE') AS tz"].first[:tz]
|
28
|
+
DB["SELECT current_timestamp() at TIME ZONE '#{tz}' AS now"].first[:now]
|
29
|
+
end
|
30
|
+
|
26
31
|
# @@integrate_with_blackstack_deployer
|
27
32
|
def self.integrate_with_blackstack_deployer()
|
28
33
|
@@integrate_with_blackstack_deployer
|
@@ -138,34 +143,6 @@ module BlackStack
|
|
138
143
|
@@jobs
|
139
144
|
end
|
140
145
|
|
141
|
-
=begin
|
142
|
-
# return a hash descriptor of the whole configuration of the cluster.
|
143
|
-
def self.to_hash()
|
144
|
-
ret = {
|
145
|
-
:log_filename => self.log_filename,
|
146
|
-
:connection_string => self.connection_string,
|
147
|
-
}
|
148
|
-
#ret[:workers] = []
|
149
|
-
#@@workers.each do |w|
|
150
|
-
# ret[:workers] << w.to_hash
|
151
|
-
#end
|
152
|
-
ret[:nodes] = []
|
153
|
-
@@nodes.each do |n|
|
154
|
-
ret[:nodes] << n.to_hash
|
155
|
-
end
|
156
|
-
ret[:jobs] = []
|
157
|
-
@@jobs.each do |j|
|
158
|
-
ret[:jobs] << j.to_hash
|
159
|
-
end
|
160
|
-
ret
|
161
|
-
end # def self.to_hash()
|
162
|
-
|
163
|
-
# setup from a whole hash descriptor
|
164
|
-
def self.initialize(h)
|
165
|
-
# TODO
|
166
|
-
end
|
167
|
-
=end
|
168
|
-
|
169
146
|
# get attached and unassigned workers.
|
170
147
|
# assign and unassign workers to jobs.
|
171
148
|
#
|
@@ -199,6 +176,7 @@ module BlackStack
|
|
199
176
|
l.logf("done (#{assigned.size.to_s})")
|
200
177
|
|
201
178
|
l.logs("Getting total pending tasks... ")
|
179
|
+
|
202
180
|
pendings = job.selecting(job.max_pending_tasks)
|
203
181
|
l.logf("done (#{pendings.size.to_s})")
|
204
182
|
|
@@ -358,7 +336,7 @@ module BlackStack
|
|
358
336
|
l.done
|
359
337
|
# kill all ruby processes except this one
|
360
338
|
l.logs("Killing all Ruby processes except this one... ")
|
361
|
-
node.
|
339
|
+
node.kill_workers()
|
362
340
|
l.done
|
363
341
|
# rename any existing folder ~/code/pampa to ~/code/pampa.<current timestamp>.
|
364
342
|
l.logs("Renaming old folder... ")
|
@@ -383,8 +361,23 @@ module BlackStack
|
|
383
361
|
# run the worker
|
384
362
|
# add these parameters for debug: debug=yes pampa=~/code/pampa/lib/pampa.rb
|
385
363
|
l.logs "Running worker #{worker.id}... "
|
386
|
-
|
387
|
-
|
364
|
+
|
365
|
+
# write bash command to initialize bash file
|
366
|
+
s = "echo \"
|
367
|
+
export RUBYLIB=$HOME/code/mysaas;
|
368
|
+
source $HOME/.profile;
|
369
|
+
source /usr/local/rvm/scripts/rvm;
|
370
|
+
cd ~/code/mysaas; rvm install 3.1.2;
|
371
|
+
rvm --default use 3.1.2;
|
372
|
+
cd #{BlackStack::Pampa.working_directory};
|
373
|
+
nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
|
374
|
+
\" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
|
375
|
+
node.exec(s, false);
|
376
|
+
|
377
|
+
#s = "nohup bash #{BlackStack::Pampa.working_directory}/worker.sh >/dev/null 2>&1 &"
|
378
|
+
s = "bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
|
379
|
+
node.exec(s, false);
|
380
|
+
|
388
381
|
l.done
|
389
382
|
}
|
390
383
|
# disconnect the node
|
@@ -418,7 +411,7 @@ module BlackStack
|
|
418
411
|
l.done
|
419
412
|
# kill all ruby processes except this one
|
420
413
|
l.logs("Killing all Ruby processes except this one... ")
|
421
|
-
node.
|
414
|
+
node.kill_workers()
|
422
415
|
l.done
|
423
416
|
# run the number of workers specified in the configuration of the Pampa module.
|
424
417
|
node.workers.each { |worker|
|
@@ -429,10 +422,18 @@ module BlackStack
|
|
429
422
|
l.logs "Running worker #{worker.id}... "
|
430
423
|
|
431
424
|
# write bash command to initialize bash file
|
432
|
-
s = "echo \"
|
425
|
+
s = "echo \"
|
426
|
+
export RUBYLIB=$HOME/code/mysaas;
|
427
|
+
source $HOME/.profile;
|
428
|
+
source /usr/local/rvm/scripts/rvm;
|
429
|
+
cd ~/code/mysaas; rvm install 3.1.2;
|
430
|
+
rvm --default use 3.1.2;
|
431
|
+
cd #{BlackStack::Pampa.working_directory};
|
432
|
+
nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
|
433
|
+
\" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
|
434
|
+
#binding.pry
|
433
435
|
node.exec(s, false);
|
434
|
-
|
435
|
-
s = "nohup bash #{BlackStack::Pampa.working_directory}/worker.sh >/dev/null 2>&1 &"
|
436
|
+
s = "nohup bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh >/dev/null 2>&1 &"
|
436
437
|
node.exec(s, false);
|
437
438
|
|
438
439
|
l.done
|
@@ -469,7 +470,7 @@ module BlackStack
|
|
469
470
|
l.done
|
470
471
|
# kill all ruby processes except this one
|
471
472
|
l.logs("Killing all Ruby processes except this one... ")
|
472
|
-
node.
|
473
|
+
node.kill_workers()
|
473
474
|
l.done
|
474
475
|
# disconnect the node
|
475
476
|
l.logs("Disconnecting... ")
|
@@ -490,7 +491,8 @@ module BlackStack
|
|
490
491
|
# connect the node
|
491
492
|
n.connect()
|
492
493
|
# get the time of the last time the worker wrote the log file
|
493
|
-
|
494
|
+
code = "cat #{BlackStack::Pampa.working_directory}/worker.#{worker_id}.log | tail -n 1 | cut -b1-19"
|
495
|
+
s = n.exec(code, false).to_s.strip
|
494
496
|
# run bash command to get the difference in minutes beteen now and the last time the worker wrote the log file
|
495
497
|
s = n.exec("echo \"$(($(date +%s) - $(date -d '#{s}' +%s))) / 60\" | bc", false).to_s.strip
|
496
498
|
# disconnect the node
|
@@ -592,6 +594,15 @@ module BlackStack
|
|
592
594
|
end
|
593
595
|
ret
|
594
596
|
end
|
597
|
+
# kill all workers
|
598
|
+
def kill_workers()
|
599
|
+
self.workers.each do |worker|
|
600
|
+
self.kill_worker(worker.id)
|
601
|
+
end
|
602
|
+
end
|
603
|
+
def kill_worker(worker_id)
|
604
|
+
self.exec("kill -9 $(ps -ef | grep \"ruby worker.rb id=#{worker_id}\" | grep -v grep | awk '{print $2}')", false)
|
605
|
+
end
|
595
606
|
end # class Node
|
596
607
|
|
597
608
|
# stub job class
|
@@ -771,14 +782,11 @@ module BlackStack
|
|
771
782
|
|
772
783
|
# returns an array of failed tasks for restarting.
|
773
784
|
def relaunching_dataset(n)
|
774
|
-
#ds = DB[self.table.to_sym].where("#{self.field_time.to_s} < CURRENT_TIMESTAMP() - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'")
|
775
|
-
#ds = ds.filter("#{self.field_end_time.to_s} IS NULL") if !self.field_end_time.nil?
|
776
|
-
#ds.limit(n).all
|
777
785
|
q = "
|
778
786
|
SELECT *
|
779
787
|
FROM #{self.table.to_s}
|
780
788
|
WHERE #{self.field_time.to_s} IS NOT NULL
|
781
|
-
AND #{self.field_time.to_s} <
|
789
|
+
AND #{self.field_time.to_s} < CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'
|
782
790
|
AND #{self.field_id.to_s} IS NOT NULL
|
783
791
|
AND #{self.field_end_time.to_s} IS NULL
|
784
792
|
AND COALESCE(#{self.field_times.to_s},0) < #{self.max_try_times.to_i}
|
@@ -802,14 +810,14 @@ module BlackStack
|
|
802
810
|
o[self.field_time.to_sym] = nil
|
803
811
|
o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
|
804
812
|
o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
|
805
|
-
|
813
|
+
o.save
|
806
814
|
end
|
807
815
|
|
808
816
|
def start(o)
|
809
817
|
if self.starter_function.nil?
|
810
|
-
o[self.field_start_time.to_sym] = DB["SELECT
|
818
|
+
o[self.field_start_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
|
811
819
|
o[self.field_times.to_sym] = o[self.field_times.to_sym].to_i + 1
|
812
|
-
|
820
|
+
o.save
|
813
821
|
else
|
814
822
|
self.starter_function.call(o, self)
|
815
823
|
end
|
@@ -817,10 +825,10 @@ module BlackStack
|
|
817
825
|
|
818
826
|
def finish(o, e=nil)
|
819
827
|
if self.finisher_function.nil?
|
820
|
-
o[self.field_end_time.to_sym] = DB["SELECT
|
828
|
+
o[self.field_end_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
|
821
829
|
o[self.field_success.to_sym] = e.nil?
|
822
830
|
o[self.field_error_description.to_sym] = e.to_console if !e.nil?
|
823
|
-
|
831
|
+
o.save
|
824
832
|
else
|
825
833
|
self.finisher_function.call(o, e, self)
|
826
834
|
end
|
@@ -853,12 +861,12 @@ module BlackStack
|
|
853
861
|
self.selecting(n).each { |o|
|
854
862
|
# count the # of dispatched
|
855
863
|
i += 1
|
856
|
-
# dispatch
|
864
|
+
# dispatch
|
857
865
|
o[self.field_id.to_sym] = worker.id
|
858
|
-
o[self.field_time.to_sym] = DB["SELECT
|
866
|
+
o[self.field_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] # IMPORTANT: use DB location to get current time.
|
859
867
|
o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
|
860
868
|
o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
|
861
|
-
|
869
|
+
o.save
|
862
870
|
# release resources
|
863
871
|
DB.disconnect
|
864
872
|
GC.start
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pampa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Leandro Daniel Sardi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-01-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sequel
|