pampa 2.0.17 → 2.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbbd19e65301b23d9e1ce78458528c5bdeecb73f2d54047e2546df8ad54f229d
-  data.tar.gz: aaea0b4f3b174b565cec083ff833c52b255aede1304757a2383f23057ea25f06
+  metadata.gz: 80cb43785569b54fd129e7ee1444f02824cf0afe85a7d231be0bfad3ab33d604
+  data.tar.gz: 85a89a2f3f050bbe35e8d2749d5db04f8f1994980a0c726917093982dacf904d
 SHA512:
-  metadata.gz: 26472faf3ef44736e06ea39351fcc4380c7ba94f282051df49f12d2572190daca659603866f6a4275dc793b1408c992fa20310500ae74d28f283d6d75eac97ee
-  data.tar.gz: 7893ba252991c1e5603b00372d5a719552a3134fdd1daed6ed168c28c9f6864ed8d36561fe531f8ffef11557c099f9340c14b8411177b96f6714e10c9f498250
+  metadata.gz: a3e1b83cfe5ab4aebd31583df38f15bc79201e0882643d7730e7ba5e3b685f90673761dfa1d51ae527861fb06c756700453da464ece938780a2231bf19dd38f2
+  data.tar.gz: a8d64fac4872f892f46d353d565d82b7faf48fbf3366d353e438948d318ed5c9a31a31ec4a22e87520a11183f9b5bc78f88bbc92a4eb6b4db39cfb669739f435

data/lib/pampa.rb

@@ -159,77 +159,80 @@ module BlackStack
           raise "The connection string is blank" if @@connection_string.strip.empty?
           # getting logger
           l = self.logger()
-          # get attached and unassigned workers 
-          l.logs "Getting attached and unassigned workers... "
-          workers = BlackStack::Pampa.workers.select { |worker| worker.attached && worker.assigned_job.nil? }
-          l.logf "done (#{workers.size.to_s})"
           # get the job this worker is working with
           BlackStack::Pampa.jobs.each { |job|
-            if workers.size == 0
-              l.logf "No more workers to assign."
-              break
-            end
-
-            l.logs("job:#{job.name}... ")
-              l.logs("Gettting assigned workers... ") 
-              assigned = BlackStack::Pampa.workers.select { |worker| worker.attached && worker.assigned_job.to_s == job.name.to_s }
-              l.logf("done (#{assigned.size.to_s})")
-
-              l.logs("Getting total pending tasks... ")
-
-              pendings = job.selecting(job.max_pending_tasks)
-              l.logf("done (#{pendings.size.to_s})")
-
-              l.logs("Has 0 tasks?.... ")
-              if pendings.size == 0
-                l.logf("yes")
-
-                l.logs("Unassigning all assigned workers... ")
-                assigned.each { |w|
-                  l.logs("Unassigning worker #{w.id}... ")
-                  w.assigned_job = nil
-                  l.done
-
-                  l.logs("Adding worker #{w.id} to the list of unassigned... ")
-                  workers << w
-                  l.done
-                }
-                l.done
-              else
-                l.logf("no")
-
-                l.logs("Reached :max_pending_tasks (#{job.max_pending_tasks}) and more than 1 assigned workers ?... ")
-                if pendings.size < job.max_pending_tasks && assigned.size > 1
-                  l.logf("no")
-
-                  l.logs("Unassigning worker... ")
-                  w = assigned.first # TODO: find a worker with no pending tasks
-                  w.assigned_job = nil
-                  l.done
+            l.log ''
+            l.logs "job #{job.name}... "
+              # get attached and unassigned workers 
+              l.logs "Getting attached and unassigned workers... "
+              workers = BlackStack::Pampa.workers.select { |w| w.attached && w.assigned_job.nil? }
+              l.logf "done (#{workers.size.to_s})"
+              # get the workers that match the filter
+              l.logs "Getting workers that match the filter... "
+              workers = workers.select { |w| w.id =~ job.filter_worker_id }
+              l.logf "done (#{workers.size.to_s})"
+              # if theere are workers
+              if workers.size > 0
+                l.logs("Gettting assigned workers... ") 
+                assigned = BlackStack::Pampa.workers.select { |worker| worker.attached && worker.assigned_job.to_s == job.name.to_s }
+                l.logf("done (#{assigned.size.to_s})")
+
+                l.logs("Getting total pending (idle) tasks... ")
+                pendings = job.idle
+                l.logf("done (#{pendings.to_s})")
+
+                l.logs("0 pending tasks?.... ")
+                if pendings.size == 0
+                  l.logf("yes")
 
-                  l.logs("Adding worker from the list of unassigned... ")
-                  workers << w
+                  l.logs("Unassigning all assigned workers... ")
+                  assigned.each { |w|
+                    l.logs("Unassigning worker... ")
+                    w.assigned_job = nil
+                    workers << w # add worker back to the list of unassigned
+                    l.logf "done (#{w.id})"
+                  }
                   l.done
                 else
-                  l.logf("yes")
+                  l.logf("no")
 
-                  l.logs("Reached :max_assigned_workers (#{job.max_assigned_workers}) and more than 0 assigned workers?... ")
-                  if assigned.size >= job.max_assigned_workers && assigned.size > 0
+                  l.logs("Under :max_pending_tasks (#{job.max_pending_tasks}) and more than 1 assigned workers ?... ")
+                  if pendings.size < job.max_pending_tasks && assigned.size > 1
                     l.logf("yes")
+
+                    while assigned.size > 1
+                      l.logs("Unassigning worker... ")
+                      w = assigned.pop # TODO: find a worker with no pending tasks
+                      w.assigned_job = nil
+                      workers << w # add worker back to the array of unassigned workers
+                      l.logf "done (#{w.id})"
+                    end
                   else
                     l.logf("no")
 
-                    l.logs("Assigning worker... ")
-                    w = workers.first
-                    w.assigned_job = job.name.to_sym
-                    l.done
-
-                    l.logs("Removing worker from the list of unassigned... ")
-                    workers.delete(w)
-                    l.done
-                  end
-                end
-              end 
+                    l.logs("Over :max_assigned_workers (#{job.max_assigned_workers}) and more than 1 assigned workers?... ")
+                    if assigned.size >= job.max_assigned_workers && assigned.size > 1
+                      l.logf("yes")
+                    else
+                      l.logf("no")
+
+                      i = assigned.size
+                      while i < job.max_assigned_workers
+                        i += 1
+                        l.logs("Assigning worker... ")
+                        w = workers.pop
+                        if w.nil?
+                          l.logf("no more workers")
+                          break
+                        else
+                          w.assigned_job = job.name.to_sym
+                          l.logf "done (#{w.id})"
+                        end
+                      end # while i < job.max_assigned_workers
+                    end # if assigned.size >= job.max_assigned_workers && assigned.size > 0
+                  end # if pendings.size < job.max_pending_tasks && assigned.size > 1
+                end # if pendings.size == 0
+              end # if workers.size > 0
             l.done
           }
         end
@@ -431,7 +434,6 @@ module BlackStack
                         cd #{BlackStack::Pampa.working_directory}; 
                         nohup ruby #{worker_filename} id=#{worker.id} config=#{self.config_filename} >/dev/null 2>&1 &
                       \" > #{BlackStack::Pampa.working_directory}/#{worker.id}.sh"
-#binding.pry
                       node.exec(s, false);
                       s = "nohup bash #{BlackStack::Pampa.working_directory}/#{worker.id}.sh >/dev/null 2>&1 &"
                       node.exec(s, false);
@@ -484,6 +486,9 @@ module BlackStack
         # connect the nodes via ssh.
         # get how many minutes the worker wrote the log file
         # close the connection
+        #
+        # DEPRECATED. Use `ps aux | grep "..."` to know if a process is running or not. 
+        #
         def self.log_minutes_ago(node_name, worker_id)
             # get the node
             n = self.nodes.select { |n| n.name == node_name }.first
@@ -659,8 +664,8 @@ module BlackStack
             # stretch assignation/unassignation of workers
             attr_accessor :max_pending_tasks
             attr_accessor :max_assigned_workers
-            # tags in order to choose on which nodes the job will be executed
-            attr_accessor :tags
+            # choose workers to assign tasks
+            attr_accessor :filter_worker_id
 
             # return a hash descriptor of the job
             def to_hash()
@@ -688,7 +693,7 @@ module BlackStack
                     :processing_function => self.processing_function.to_s,
                     :max_pending_tasks => self.max_pending_tasks,
                     :max_assigned_workers => self.max_assigned_workers,
-                    :tags => self.tags
+                    :filter_worker_id => self.filter_worker_id
                 }
             end
 
@@ -724,7 +729,7 @@ module BlackStack
               self.processing_function = h[:processing_function]
               self.max_pending_tasks = h[:max_pending_tasks]
               self.max_assigned_workers = h[:max_assigned_workers]
-              self.tags = h[:tags].nil? ? [] : ( h[:tags].is_a?(Array) ? h[:tags] : [h[:tags].to_s] )
+              self.filter_worker_id = h[:filter_worker_id]
             end
             
             # returns an array of tasks pending to be processed by the worker.
@@ -806,7 +811,17 @@ module BlackStack
             end
             
             def update(o)
-              DB[self.table.to_sym].where(
+              # use the select to update ONLY the pampa fields.
+              DB[self.table.to_sym].select(
+                :field_primary_key,
+                :field_id,
+                :field_time,
+                :field_times,
+                :field_start_time,
+                :field_end_time,
+                :field_success,
+                :field_error_description
+              ).where(
                 self.field_primary_key.to_sym => o[self.field_primary_key.to_sym]
               ).update(o)
             end
@@ -864,69 +879,96 @@ module BlackStack
               # dispatching n pending records
               i = 0
               if n>0
-                self.selecting(n).each { |o|
-                  # count the # of dispatched
-                  i += 1
-                  # dispatch 
-                  o[self.field_id.to_sym] = worker.id
-                  o[self.field_time.to_sym] = DB["SELECT CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP) AS dt"].first[:dt] # IMPORTANT: use DB location to get current time.
-                  o[self.field_start_time.to_sym] = nil if !self.field_start_time.nil?
-                  o[self.field_end_time.to_sym] = nil if !self.field_end_time.nil?
-                  self.update(o)
-                  # release resources
-                  DB.disconnect
-                  GC.start        
-                }
+                ids = self.selecting(n).map { |h| h[:id] }
+
+                i = ids.size
+                q = "
+                  UPDATE #{self.table.to_s}
+                  SET 
+                    #{self.field_id.to_s} = '#{worker.id}', 
+                "
+
+                if !self.field_start_time.nil?
+                  q += "
+                  #{self.field_start_time.to_s} = NULL,
+                  "
+                end
+
+                if !self.field_end_time.nil?
+                  q += "
+                  #{self.field_end_time.to_s} = NULL,
+                  "                  
+                end
+
+                q += "
+                  #{self.field_time.to_s} = CAST('#{BlackStack::Pampa.now}' AS TIMESTAMP)  
+                  WHERE #{self.field_primary_key.to_s} IN ('#{ids.join("','")}')
+                "
+
+                DB.execute(q)
               end
               
               #      
               return i
             end
 
+# reporting methods
+# idle + failed + completed = total
+
             # reporting method: idle
             # reutrn the number of idle tasks.
             # if the numbr if idle tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
-            def idle(max_tasks_to_show=25)
-                j = self
-                ret = j.selecting(max_tasks_to_show).size
-                ret = ret >= max_tasks_to_show ? "#{ret.to_label}+" : ret.to_label
-                ret
-            end # def idle
+            def total
+              j = self
+              q = "
+                  SELECT COUNT(*) AS n
+                  FROM #{j.table.to_s} 
+              "
+              DB[q].first[:n].to_i
+            end # def total
 
-            # reporting method: running
-            # return the number of running tasks.
-            # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
-            def running(max_tasks_to_show=25)
-                j = self
-                ret = 0
-                BlackStack::Pampa::nodes.each { |n|
-                    n.workers.each { |w|
-                        ret += j.occupied_slots(w).size
-                        break if ret>=max_tasks_to_show
-                    }
-                }
-                ret = ret >= max_tasks_to_show ? "#{max_tasks_to_show}+" : ret.to_label
-                ret
-            end # def idle
 
-            # reporting method: running
-            # return the number of running tasks.
-            # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
-            def failed(max_tasks_to_show=25)
+            # reporting method: idle
+            # reutrn the number of idle tasks.
+            # if the numbr if idle tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            def completed
+              j = self
+              q = "
+                  SELECT COUNT(*) AS n
+                  FROM #{j.table.to_s} 
+                  WHERE COALESCE(#{j.field_success.to_s},false)=true
+              "
+              DB[q].first[:n].to_i
+            end # def completed
+
+            # reporting method: idle
+            # reutrn the number of idle tasks.
+            # if the numbr if idle tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            def idle
                 j = self
                 q = "
-                    SELECT * 
+                    SELECT COUNT(*) AS n
                     FROM #{j.table.to_s} 
-                    WHERE COALESCE(#{j.field_success.to_s},true)=false
-                    --AND #{j.field_end_time.to_s} IS NULL
-                    --AND COALESCE(#{j.field_times.to_s},0) >= #{j.max_try_times.to_i}
-                    LIMIT #{max_tasks_to_show}
+                    WHERE COALESCE(#{j.field_success.to_s},false)=false
+                    AND COALESCE(#{j.field_times.to_s},0) < #{j.max_try_times.to_i}
                 "
-                ret = DB[q].all.size
-                ret = ret >= max_tasks_to_show ? "#{ret.to_label}+" : ret.to_label
-                ret
+                DB[q].first[:n].to_i
             end # def idle
 
+            # reporting method: running
+            # return the number of running tasks.
+            # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            def failed
+              j = self
+              q = "
+                  SELECT COUNT(*) AS n
+                  FROM #{j.table.to_s} 
+                  WHERE COALESCE(#{j.field_success.to_s},false)=false
+                  AND COALESCE(#{j.field_times.to_s},0) >= #{j.max_try_times.to_i}
+              "
+              DB[q].first[:n].to_i
+            end # def falsed
+
             # reporting method: error_descriptions
             # return an array of hashes { :id, :error_description } with the tasks that have an the success flag in false, error description.
             # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show` errors.
@@ -936,8 +978,6 @@ module BlackStack
                     SELECT #{j.field_primary_key.to_s} as id, #{j.field_error_description.to_s} as description 
                     FROM #{j.table.to_s} 
                     WHERE COALESCE(#{j.field_success.to_s},true)=false
-                    --AND #{j.field_end_time.to_s} IS NULL
-                    --AND COALESCE(#{j.field_times.to_s},0) >= #{j.max_try_times.to_i}
                     LIMIT #{max_tasks_to_show}
                 "
                 DB[q].all

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pampa
 version: !ruby/object:Gem::Version
-  version: 2.0.17
+  version: 2.0.18
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-18 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel