pampa 2.0.27 → 2.0.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 766a4c8e58dcd60e3938ae32600b3715f696418c7f4f2d4cb15ea22fa8559213
-  data.tar.gz: 2a131d44bba1996839e5fc03a0a12148c6429d49b050a711c4bcc91a00fe34b2
+  metadata.gz: '095438d108ee4f3cd1cf5d329fb1a0d29c38d6647a130c811e75ed2d73105d81'
+  data.tar.gz: 45dd332d64c86e75d7e0b0f8e4575e3884a9cf486eabb959371af3b1da2c674c
 SHA512:
-  metadata.gz: 3e8bc763b08173e152f9a7a6fdc3c75fa1aca6e2522cba7b919fa2b5c6f71653266fc7f2e29fbfc749a59af706bf3cb63624ef8ce4da903349ae7001c4c5aad1
-  data.tar.gz: d03a80fcd845fe760be93a42b720528cc518fd1380cee51dc1d1ab2439254d23ed2ef6c42acf4075b9290f513151486b28c90bf7c568731516a29394a1859db9
+  metadata.gz: e76a96906f54dafa73f37c4109581f1779c77f086788a98e8278960bf2086228a1fd09851b3e37b94c72920495d009567d1f96699c63b87567925c84b8a1af2c
+  data.tar.gz: a4869784e7f39a12eb3ef8327584ab773a0a0eac2b22c997cde5dd008e34f13fcdc5e9c0b05b5db86392e3fe8ffa4f8fdd5699052c5e5781d9a9990cae0546d9

data/lib/pampa.rb

@@ -1,14 +1,11 @@
 require 'sequel'
 require 'blackstack-core'
 require 'blackstack-nodes'
-require 'blackstack-deployer'
 require 'simple_command_line_parser'
 require 'simple_cloud_logging'
 
 module BlackStack
     module Pampa
-        # activate this flag if you want to add pampa nodes to blackstack-deployer.
-        @@integrate_with_blackstack_deployer = false 
         # setup custom locations for config and worker files.
         @@config_filename = "config.rb"
         @@worker_filename = "worker.rb"
@@ -28,15 +25,6 @@ module BlackStack
           DB["SELECT current_timestamp at TIME ZONE '#{tz}' AS now"].first[:now]
         end
 
-        # @@integrate_with_blackstack_deployer
-        def self.integrate_with_blackstack_deployer()
-            @@integrate_with_blackstack_deployer
-        end
-
-        def self.set_integrate_with_blackstack_deployer(b)
-            @@integrate_with_blackstack_deployer = b
-        end
-
         # @@config_filename
         def self.config_filename()
             @@config_filename
@@ -97,8 +85,6 @@ module BlackStack
         # add a node to the cluster.
         def self.add_node(h)
             @@nodes << BlackStack::Pampa::Node.new(h)
-            # add to deployer
-            BlackStack::Deployer.add_node(h) if @@integrate_with_blackstack_deployer
         end # def self.add_node(h)
 
         # add an array of nodes to the cluster.
@@ -177,8 +163,8 @@ module BlackStack
                 assigned = BlackStack::Pampa.workers.select { |worker| worker.attached && worker.assigned_job.to_s == job.name.to_s }
                 l.logf("done (#{assigned.size.to_s})")
 
-                l.logs("Getting total pending (idle) tasks... ")
-                pendings = job.idle
+                l.logs("Getting total pending (pending) tasks... ")
+                pendings = job.pending
                 l.logf("done (#{pendings.to_s})")
 
                 l.logs("0 pending tasks?.... ")
@@ -632,6 +618,9 @@ module BlackStack
             # max number of times that a record can start to process & fail (:start_time field is not nil, 
             # but :end_time field is still nil after :max_job_duration_minutes)
             attr_accessor :max_try_times
+
+            # CUSTOM DISPATCHING FUNCTIONS
+            # 
             # additional function to returns an array of tasks pending to be processed by a worker.
             # it should returns an array
             # keep it nil if you want to run the default function
@@ -661,9 +650,23 @@ module BlackStack
             attr_accessor :finisher_function
             # Function to execute for each task.
             attr_accessor :processing_function
+
+            # CUSTOM REPORTING FUNCTIONS
+            # 
+            # additional function to returns the number of total tasks.
+            # it should returns an array
+            # keep it nil if you want to run the default function
+            attr_accessor :total_function
+            attr_accessor :completed_function
+            attr_accessor :pending_function
+            attr_accessor :failed_function
+
+            # ELASTIC WORKERS ASSIGNATION
+            # 
             # stretch assignation/unassignation of workers
             attr_accessor :max_pending_tasks
             attr_accessor :max_assigned_workers
+
             # choose workers to assign tasks
             attr_accessor :filter_worker_id
 
@@ -683,6 +686,8 @@ module BlackStack
                     :queue_size => self.queue_size,
                     :max_job_duration_minutes => self.max_job_duration_minutes,
                     :max_try_times => self.max_try_times,
+
+                    # dispatching custom functions
                     :occupied_function => self.occupied_function.to_s,
                     :allowing_function => self.allowing_function.to_s,
                     :selecting_function => self.selecting_function.to_s,
@@ -693,7 +698,13 @@ module BlackStack
                     :processing_function => self.processing_function.to_s,
                     :max_pending_tasks => self.max_pending_tasks,
                     :max_assigned_workers => self.max_assigned_workers,
-                    :filter_worker_id => self.filter_worker_id
+                    :filter_worker_id => self.filter_worker_id,
+
+                    # reporting custom functions
+                    :total_function => self.total_function.to_s,
+                    :completed_function => self.completed_function.to_s,
+                    :pending_function => self.pending_function.to_s,
+                    :failed_function => self.failed_function.to_s,
                 }
             end
 
@@ -721,12 +732,22 @@ module BlackStack
               self.queue_size = h[:queue_size]
               self.max_job_duration_minutes = h[:max_job_duration_minutes]  
               self.max_try_times = h[:max_try_times]
+
+              # dispatching custom functions
               self.occupied_function = h[:occupied_function]
               self.allowing_function = h[:allowing_function]
               self.selecting_function = h[:selecting_function]
               self.relaunching_function = h[:relaunching_function]
               self.relauncher_function = h[:relauncher_function]
               self.processing_function = h[:processing_function]
+
+              # reporting custom functions
+              self.total_function = h[:total_function]
+              self.completed_function = h[:completed_function]
+              self.pending_function = h[:pending_function]
+              self.failed_function = h[:failed_function]
+
+              # elastic workers assignation
               self.max_pending_tasks = h[:max_pending_tasks]
               self.max_assigned_workers = h[:max_assigned_workers]
               self.filter_worker_id = h[:filter_worker_id]
@@ -914,62 +935,112 @@ module BlackStack
               return i
             end
 
-# reporting methods
-# idle + failed + completed = total
+            # reporting methods
+            # 
 
-            # reporting method: idle
-            # reutrn the number of idle tasks.
-            # if the numbr if idle tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            # reporting method: total
+            # reutrn the number of total tasks.
+            # if the numbr if total tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
             def total
               j = self
-              q = "
-                  SELECT COUNT(*) AS n
-                  FROM #{j.table.to_s} 
-              "
-              DB[q].first[:n].to_i
+              if self.total_function.nil?
+                q = "
+                    SELECT COUNT(*) AS n
+                    FROM #{j.table.to_s} 
+                "
+                return DB[q].first[:n].to_i
+              else
+                return self.total_function.call
+              end
             end # def total
 
 
-            # reporting method: idle
-            # reutrn the number of idle tasks.
-            # if the numbr if idle tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            # reporting method: completed
+            # reutrn the number of completed tasks.
+            # if the numbr if completed tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
             def completed
               j = self
-              q = "
-                  SELECT COUNT(*) AS n
-                  FROM #{j.table.to_s} 
-                  WHERE COALESCE(#{j.field_success.to_s},false)=true
-              "
-              DB[q].first[:n].to_i
+              if self.completed_function.nil?
+                q = "
+                    SELECT COUNT(*) AS n
+                    FROM #{j.table.to_s} 
+                    WHERE COALESCE(#{j.field_success.to_s},false)=true
+                "
+                return DB[q].first[:n].to_i
+              else
+                return self.completed_function.call
+              end
             end # def completed
 
-            # reporting method: idle
-            # reutrn the number of idle tasks.
-            # if the numbr if idle tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
-            def idle
+            # reporting method: pending
+            # reutrn the number of pending tasks.
+            # if the numbr if pending tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            def pending
                 j = self
+                if self.pending_function.nil?
+                  q = "
+                      SELECT COUNT(*) AS n
+                      FROM #{j.table.to_s} 
+                      WHERE COALESCE(#{j.field_success.to_s},false)=false
+                      AND COALESCE(#{j.field_times.to_s},0) < #{j.max_try_times.to_i}
+                  "
+                  return DB[q].first[:n].to_i
+                else
+                  return self.pending_function.call
+                end
+            end # def pending
+
+            # reporting method: running
+            # return the number of running tasks.
+            # if the number if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
+            def failed
+              j = self
+              if self.failed_function.nil?
                 q = "
                     SELECT COUNT(*) AS n
                     FROM #{j.table.to_s} 
                     WHERE COALESCE(#{j.field_success.to_s},false)=false
-                    AND COALESCE(#{j.field_times.to_s},0) < #{j.max_try_times.to_i}
+                    AND COALESCE(#{j.field_times.to_s},0) >= #{j.max_try_times.to_i}
                 "
-                DB[q].first[:n].to_i
-            end # def idle
+                return DB[q].first[:n].to_i
+              else
+                return self.failed_function.call
+              end
+            end # def falsed
 
-            # reporting method: running
-            # return the number of running tasks.
+            # reporting method: timeline
+            # Return an array of hashes with the number of successfull processed taasks in the last period.
+            # The period is defined by the `scale_unit` and `scale_points` parameters.
+            # The `scale_unit` can be `minutes`, `hours`, `days`, `weeks`, `months`, `years`.
+            # The `scale_points` is the number of `scale_unit` to be reported, and it must be an integer higer than 0.
             # if the numbr if running tasks is higher than `max_tasks_to_show` then it returns `max_tasks_to_show`+.
-            def failed
+            def timeline(scale_unit='minutes', scale_points=60)
               j = self
-              q = "
+              a = []
+              # validate: The period is defined by the `scale_unit` and `scale_points` parameters.
+              if !['minutes', 'hours', 'days', 'weeks', 'months', 'years'].include?(scale_unit)
+                raise "Invalid scale_unit: #{scale_unit}"
+              end
+              # validate: The `scale_points` is the number of `scale_unit` to be reported, and it must be an integer higer than 0.
+              if !scale_points.is_a?(Integer) || scale_points<=0
+                raise "Invalid scale_points: #{scale_points}"
+              end
+              # generate report
+              point = 0
+              while point<scale_points
+                point += 1
+                q = "
                   SELECT COUNT(*) AS n
-                  FROM #{j.table.to_s} 
-                  WHERE COALESCE(#{j.field_success.to_s},false)=false
-                  AND COALESCE(#{j.field_times.to_s},0) >= #{j.max_try_times.to_i}
-              "
-              DB[q].first[:n].to_i
-            end # def falsed
+                  FROM #{j.table.to_s}
+                  WHERE COALESCE(#{j.field_success.to_s},false)=true
+                  AND #{j.field_time.to_s} >= CAST('#{BlackStack::Pampa.now - point.send(scale_unit)}' AS TIMESTAMP)
+                  AND #{j.field_time.to_s} < CAST('#{BlackStack::Pampa.now - (point-1).send(scale_unit)}' AS TIMESTAMP)
+                "
+                a << { :time => BlackStack::Pampa.now - (point-1).send(scale_unit), :n => DB[q].first[:n].to_i }
+              end # while point<scale_points
+              # return
+              a
+            end # def timeline
 
             # reporting method: error_descriptions
             # return an array of hashes { :id, :error_description } with the tasks that have an the success flag in false, error description.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pampa
 version: !ruby/object:Gem::Version
-  version: 2.0.27
+  version: 2.0.29
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-17 00:00:00.000000000 Z
+date: 2023-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -70,26 +70,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.11
-- !ruby/object:Gem::Dependency
-  name: blackstack-deployer
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.24
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.24
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.2.24
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.24
 - !ruby/object:Gem::Dependency
   name: simple_command_line_parser
   requirement: !ruby/object:Gem::Requirement