pampa 2.0.14 → 2.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0137672f5c0b7c0cd2be75ddf221d84d65d824c8ba4bdf87b5cfd369bf9eef14
-  data.tar.gz: 7d8bb53cfd11c3d37b69b8408c544f1718fc0178f130f5f16785771e84af9ac9
+  metadata.gz: 5f8fec0d0fcba61ab662150060b8ce14185a9d3c3d2d4a74a3c255da0ae3e8e6
+  data.tar.gz: d290f6eb64265a7a16451ea4b6e9be99f6e3ca4f86493703894b44fc8f57c275
 SHA512:
-  metadata.gz: '00540329934fd5d5d22cae40967fee37ba4eba39a4383d324999da4902ea0a8e83c489a320820102ec3c8175a37fdace4a2147b0be490ec22e6c8380ab29f1a2'
-  data.tar.gz: 4efb7d405e0aa5d6a7c6f8b613b83fe14511d2f15a14f9338c48da4676204631a0c99fc34491474571c164779ad730e5820b397f2dc93a3abb3ff5bee6ec1851
+  metadata.gz: 49ccc2d9c1440d65ffd0b137425807ff0e34133363d7308f3a9d4fea050d24d078a319f608f0bb3a0f68622154603af4f3ec6f858ea83f743789d0d3b6cd1a2f
+  data.tar.gz: 49b313fa4e0ef847643e632249cf529dfdc0ecc357de0fac1f226694895f3959dc5e777002c9481ecff4946d6638e46184b64f919fae97f7ed5bb49785269bfe

data/lib/pampa.rb

@@ -23,6 +23,11 @@ module BlackStack
         # Connection string to the database. Example: mysql2://user:password@localhost:3306/database
         @@connection_string = nil
 
+        def self.now()
+          tz = 'America/Argentina/Buenos_Aires' #DB["SELECT current_setting('TIMEZONE') AS tz"].first[:tz]
+          DB["SELECT current_timestamp() at TIME ZONE '#{tz}' AS now"].first[:now]
+        end
+
         # @@integrate_with_blackstack_deployer
         def self.integrate_with_blackstack_deployer()
             @@integrate_with_blackstack_deployer
@@ -138,34 +143,6 @@ module BlackStack
           @@jobs
         end
 
-=begin
-        # return a hash descriptor of the whole configuration of the cluster.
-        def self.to_hash()
-            ret = {
-                :log_filename => self.log_filename,
-                :connection_string => self.connection_string,
-            }
-            #ret[:workers] = []
-            #@@workers.each do |w|
-            #    ret[:workers] << w.to_hash
-            #end
-            ret[:nodes] = []
-            @@nodes.each do |n|
-                ret[:nodes] << n.to_hash
-            end
-            ret[:jobs] = []
-            @@jobs.each do |j|
-                ret[:jobs] << j.to_hash
-            end
-            ret
-        end # def self.to_hash()
-
-        # setup from a whole hash descriptor
-        def self.initialize(h)
-            # TODO
-        end
-=end
-
         # get attached and unassigned workers. 
         # assign and unassign workers to jobs.
         #
@@ -199,6 +176,7 @@ module BlackStack
               l.logf("done (#{assigned.size.to_s})")
 
               l.logs("Getting total pending tasks... ")
+
               pendings = job.selecting(job.max_pending_tasks)
               l.logf("done (#{pendings.size.to_s})")
 
@@ -358,7 +336,7 @@ module BlackStack
                     l.done
                     # kill all ruby processes except this one
                     l.logs("Killing all Ruby processes except this one... ")
-                    node.exec("ps ax | grep ruby | grep -v grep | grep -v #{Process.pid} | cut -b3-7 | xargs -t kill;", false);
+                    node.kill_workers()
                     l.done
                     # rename any existing folder ~/code/pampa to ~/code/pampa.<current timestamp>.
                     l.logs("Renaming old folder... ")
@@ -383,8 +361,23 @@ module BlackStack
                         # run the worker
                         # add these parameters for debug: debug=yes pampa=~/code/pampa/lib/pampa.rb
                         l.logs "Running worker #{worker.id}... "
-                        s = "nohup ruby #{BlackStack::Pampa.worker_filename} id=#{worker.id} config=#{BlackStack::Pampa.working_directory}/#{BlackStack::Pampa.config_filename} >/dev/null 2>&1 &" 
-                        node.exec("#{s}", false);
+
+                        # write bash command to initialize bash file
+                        s = "echo \"
+                          export RUBYLIB=$HOME/code/mysaas;
+                          source $HOME/.profile; 
+                          source /usr/local/rvm/scripts/rvm;
+                          cd ~/code/mysaas; rvm install 3.1.2;
+                          rvm --default use 3.1.2;
+                          cd #{BlackStack::Pampa.working_directory}; 
+                          nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
+                        \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
+                        node.exec(s, false);
+
+                        #s = "nohup bash #{BlackStack::Pampa.working_directory}/worker.sh >/dev/null 2>&1 &"
+                        s = "bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
+                        node.exec(s, false);
+
                         l.done
                     }
                     # disconnect the node
@@ -418,7 +411,7 @@ module BlackStack
                   l.done
                   # kill all ruby processes except this one
                   l.logs("Killing all Ruby processes except this one... ")
-                  node.exec("ps ax | grep ruby | grep -v grep | grep -v #{Process.pid} | cut -b3-7 | xargs -t kill;", true);
+                  node.kill_workers()
                   l.done
                   # run the number of workers specified in the configuration of the Pampa module.
                   node.workers.each { |worker|
@@ -429,10 +422,18 @@ module BlackStack
                       l.logs "Running worker #{worker.id}... "
 
                       # write bash command to initialize bash file
-                      s = "echo \"source $HOME/.profile; cd #{BlackStack::Pampa.working_directory}; nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &\" > #{BlackStack::Pampa.working_directory}/worker.sh"
+                      s = "echo \"
+                        export RUBYLIB=$HOME/code/mysaas;
+                        source $HOME/.profile; 
+                        source /usr/local/rvm/scripts/rvm;
+                        cd ~/code/mysaas; rvm install 3.1.2;
+                        rvm --default use 3.1.2;
+                        cd #{BlackStack::Pampa.working_directory}; 
+                        nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
+                      \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
+#binding.pry
                       node.exec(s, false);
-                    
-                      s = "nohup bash #{BlackStack::Pampa.working_directory}/worker.sh >/dev/null 2>&1 &"
+                      s = "nohup bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh >/dev/null 2>&1 &"
                       node.exec(s, false);
 
                       l.done
@@ -469,7 +470,7 @@ module BlackStack
                     l.done
                     # kill all ruby processes except this one
                     l.logs("Killing all Ruby processes except this one... ")
-                    node.exec("ps ax | grep ruby | grep -v grep | grep -v #{Process.pid} | cut -b3-7 | xargs -t kill;", false);
+                    node.kill_workers()
                     l.done
                     # disconnect the node
                     l.logs("Disconnecting... ")
@@ -490,7 +491,8 @@ module BlackStack
             # connect the node
             n.connect()
             # get the time of the last time the worker wrote the log file
-            s = n.exec("cat #{BlackStack::Pampa.working_directory}/worker.#{worker_id}.log | tail -n 1 | cut -b1-19", false).to_s.strip
+            code = "cat #{BlackStack::Pampa.working_directory}/worker.#{worker_id}.log | tail -n 1 | cut -b1-19"
+            s = n.exec(code, false).to_s.strip
             # run bash command to get the difference in minutes beteen now and the last time the worker wrote the log file
             s = n.exec("echo \"$(($(date +%s) - $(date -d '#{s}' +%s))) / 60\" | bc", false).to_s.strip
             # disconnect the node
@@ -592,6 +594,15 @@ module BlackStack
                 end
                 ret
             end
+            # kill all workers
+            def kill_workers()
+                self.workers.each do |worker|
+                    self.kill_worker(worker.id)
+                end
+            end
+            def kill_worker(worker_id)
+                self.exec("kill -9 $(ps -ef | grep \"ruby worker.rb id=#{worker_id}\" | grep -v grep | awk '{print $2}')", false)
+            end
         end # class Node
 
         # stub job class
@@ -771,14 +782,11 @@ module BlackStack
         
             # returns an array of failed tasks for restarting.
             def relaunching_dataset(n)
-              #ds = DB[self.table.to_sym].where("#{self.field_time.to_s} < CURRENT_TIMESTAMP() - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes'")
-              #ds = ds.filter("#{self.field_end_time.to_s} IS NULL") if !self.field_end_time.nil?  
-              #ds.limit(n).all
               q = "
                 SELECT * 
                 FROM #{self.table.to_s} 
                 WHERE #{self.field_time.to_s} IS NOT NULL 
-                AND #{self.field_time.to_s} < CURRENT_TIMESTAMP() - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes' 
+                AND #{self.field_time.to_s} < CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) - INTERVAL '#{self.max_job_duration_minutes.to_i} minutes' 
                 AND #{self.field_id.to_s} IS NOT NULL 
                 AND #{self.field_end_time.to_s} IS NULL
                 AND COALESCE(#{self.field_times.to_s},0) < #{self.max_try_times.to_i}
@@ -802,14 +810,14 @@ module BlackStack
               o[self.field_time.to_sym] = nil
               o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
               o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
-              DB[self.table.to_sym].where(self.field_primary_key.to_sym => o[self.field_primary_key.to_sym]).update(o)
+              o.save
             end
         
             def start(o)
               if self.starter_function.nil?
-                o[self.field_start_time.to_sym] = DB["SELECT CURRENT_TIMESTAMP() AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
+                o[self.field_start_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_start_time.nil? # IMPORTANT: use DB location to get current time.
                 o[self.field_times.to_sym] = o[self.field_times.to_sym].to_i + 1
-                DB[self.table.to_sym].where(self.field_primary_key.to_sym => o[self.field_primary_key.to_sym]).update(o)
+                o.save
               else
                 self.starter_function.call(o, self)
               end
@@ -817,10 +825,10 @@ module BlackStack
         
             def finish(o, e=nil)
               if self.finisher_function.nil?
-                o[self.field_end_time.to_sym] = DB["SELECT CURRENT_TIMESTAMP() AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
+                o[self.field_end_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] if !self.field_end_time.nil? && e.nil? # IMPORTANT: use DB location to get current time.
                 o[self.field_success.to_sym] = e.nil?
                 o[self.field_error_description.to_sym] = e.to_console if !e.nil? 
-                DB[self.table.to_sym].where(self.field_primary_key.to_sym => o[self.field_primary_key.to_sym]).update(o)
+                o.save
               else
                 self.finisher_function.call(o, e, self)
               end
@@ -853,12 +861,12 @@ module BlackStack
                 self.selecting(n).each { |o|
                   # count the # of dispatched
                   i += 1
-                  # dispatch records
+                  # dispatch 
                   o[self.field_id.to_sym] = worker.id
-                  o[self.field_time.to_sym] = DB["SELECT CURRENT_TIMESTAMP() AS dt"].first[:dt] # IMPORTANT: use DB location to get current time.
+                  o[self.field_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] # IMPORTANT: use DB location to get current time.
                   o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
                   o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
-                  DB[self.table.to_sym].where(self.field_primary_key.to_sym => o[self.field_primary_key.to_sym]).update(o)
+                  o.save
                   # release resources
                   DB.disconnect
                   GC.start        

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pampa
 version: !ruby/object:Gem::Version
-  version: 2.0.14
+  version: 2.0.16
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-11 00:00:00.000000000 Z
+date: 2022-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel