pampa 2.0.29 → 2.0.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '095438d108ee4f3cd1cf5d329fb1a0d29c38d6647a130c811e75ed2d73105d81'
-  data.tar.gz: 45dd332d64c86e75d7e0b0f8e4575e3884a9cf486eabb959371af3b1da2c674c
+  metadata.gz: 8610d7df9639c7fe8e09b420795dc61c0cdb76b4df04b04a935689bec1d3d5be
+  data.tar.gz: 2c9d0b02219822ca36091805ab2e71ea59285db83255da737acd84d23eda7414
 SHA512:
-  metadata.gz: e76a96906f54dafa73f37c4109581f1779c77f086788a98e8278960bf2086228a1fd09851b3e37b94c72920495d009567d1f96699c63b87567925c84b8a1af2c
-  data.tar.gz: a4869784e7f39a12eb3ef8327584ab773a0a0eac2b22c997cde5dd008e34f13fcdc5e9c0b05b5db86392e3fe8ffa4f8fdd5699052c5e5781d9a9990cae0546d9
+  metadata.gz: 4500f05166c288b46cdce8f6c361cc9ca9a26314300b1e97248474ac2be432d863b2e34e0e3b24398c677910ef109ca0cbdeddf65d5fc387227560806966d784
+  data.tar.gz: e364de99b69c3c93b30fb59f69450a5e8637e91f36d8b810fb7eeb19b084c1686362841a48a1118aadc2fe347f31b0a3c2fc370754cbe536d941d1f770a2de4d

data/lib/pampa/app.rb

@@ -0,0 +1,194 @@
+# MySaaS - Pampa Dashboard
+# Copyright (C) 2022 ExpandedVenture, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+#
+# Authors: Leandro Daniel Sardi (https://github.com/leandrosardi)
+#
+
+require 'pampa'
+require "rubygems"
+
+# 
+PARSER = BlackStack::SimpleCommandLineParser.new(
+  :description => 'This command will launch a Sinatra-based Pampa dashboard.', 
+  :configuration => [{
+    :name=>'port', 
+    :mandatory=>false, 
+    :description=>'Listening port. Default: 3000.', 
+    :type=>BlackStack::SimpleCommandLineParser::INT,
+    :default => 3000,
+  }, {
+    :name=>'config', 
+    :mandatory=>false, 
+    :description=>'Configuration file. Default: "config.rb".', 
+    :type=>BlackStack::SimpleCommandLineParser::STRING,
+    :default => 'config.rb',
+  }, {
+    :name=>'db',
+    :mandatory=>false,
+    :default=>'postgres', 
+    :description=>'Database driver. Supported values: postgres, crdb. Default: postgres.', 
+    :type=>BlackStack::SimpleCommandLineParser::STRING,
+  }, {
+    :name=>'log',
+    :mandatory=>false,
+    :default=>true, 
+    :description=>'If write log in the file ./app.log or not. Default: "yes"', 
+    :type=>BlackStack::SimpleCommandLineParser::BOOL,
+  }]
+)
+
+# create logger
+l = PARSER.value('log') ? BlackStack::LocalLogger.new('app.log') : BlackStack::BaseLogger.new(nil)
+
+#
+# load config file
+l.logs "Loading #{PARSER.value('config').to_s.blue}... "
+require PARSER.value('config')
+l.logf 'done'.green
+
+l.logs 'Connecting to database... '
+if PARSER.value('db') == 'postgres'
+    DB = BlackStack::PostgreSQL::connect
+elsif PARSER.value('db') == 'crdb'
+    DB = BlackStack::CockroachDB::connect
+else
+    raise 'Unknown database driver.'
+end
+l.logf 'done'.green
+
+# 
+spec = Gem.loaded_specs['pampa']
+puts '
+_______  _______  __   __  _______  _______ 
+|       ||   _   ||  |_|  ||       ||   _   |
+|    _  ||  |_|  ||       ||    _  ||  |_|  |
+|   |_| ||       ||       ||   |_| ||       |
+|    ___||       ||       ||    ___||       |
+|   |    |   _   || ||_|| ||   |    |   _   |
+|___|    |__| |__||_|   |_||___|    |__| |__|
+
+Version: '+spec.version.to_s.green+'.
+Authors: '+spec.authors.join(', ').green+'.
+Documentation: '+spec.homepage.blue+'
+
+Sandbox: '+ (BlackStack.sandbox? ? 'yes'.green : 'no'.yellow) +'
+
+'
+
+PORT = PARSER.value("port")
+
+configure { set :server, :puma }
+set :bind, '0.0.0.0'
+set :port, PORT
+enable :sessions
+enable :static
+
+configure do
+  enable :cross_origin
+end  
+
+before do
+  headers 'Access-Control-Allow-Origin' => '*', 
+          'Access-Control-Allow-Methods' => ['OPTIONS', 'GET', 'POST']  
+end
+
+set :protection, false
+
+# Setting the root of views and public folders in the `~/code` folder in order to have access to extensions.
+# reference: https://stackoverflow.com/questions/69028408/change-sinatra-views-directory-location
+set :root,  '.'
+binding.pry
+set :views, Gem.loaded_specs['pampa'].full_gem_path
+
+# page not found redirection
+not_found do
+  redirect '/404'
+end
+
+# unhandled exception redirectiopn
+error do
+  max_lenght = 8000
+  s = "message=#{CGI.escape(env['sinatra.error'].to_s)}&"
+  s += "backtrace_size=#{CGI.escape(env['sinatra.error'].backtrace.size.to_s)}&"
+  i = 0
+  env['sinatra.error'].backtrace.each { |a| 
+    a = "backtrace[#{i.to_s}]=#{CGI.escape(a.to_s)}&"
+    and_more = "backtrace[#{i.to_s}]=..." 
+    if (s+a).size > max_lenght - and_more.size
+      s += and_more
+      break
+    else
+      s += a
+    end
+    i += 1 
+  }
+  redirect "/500?#{s}"
+end
+
+# condition: api_key parameter is required too for the access points
+set(:api_key) do |*roles|
+  condition do
+    @return_message = {}
+    
+    @return_message[:status] = 'success'
+
+    # validate: the pages using the :api_key condition must work as post only.
+    if request.request_method != 'POST'
+      @return_message[:status] = 'Pages with an `api_key` parameter are only available for POST requests.'
+      @return_message[:value] = ""
+      halt @return_message.to_json
+    end
+
+    @body = JSON.parse(request.body.read)
+
+    if !@body.has_key?('api_key')
+      # libero recursos
+      DB.disconnect 
+      GC.start
+      @return_message[:status] = "api_key is required on #{@body.to_s}"
+      @return_message[:value] = ""
+      halt @return_message.to_json
+    end
+
+    if !@body['api_key'].guid?
+      # libero recursos
+      DB.disconnect 
+      GC.start
+  
+      @return_message[:status] = "Invalid api_key (#{@body['api_key']}))"
+      @return_message[:value] = ""
+      halt @return_message.to_json      
+    end
+    
+    validation_api_key = @body['api_key'].to_guid.downcase
+
+    if validation_api_key != API_KEY
+      # libero recursos
+      DB.disconnect 
+      GC.start
+      #     
+      @return_message[:status] = 'Wrong api_key'
+      @return_message[:value] = ""
+      halt @return_message.to_json        
+    end
+  end
+end
+
+get '/404', :agent => /(.*)/ do
+  erb :'views/404', :layout => :'/views/layouts/public'
+end
+
+get '/500', :agent => /(.*)/ do
+  erb :'views/500', :layout => :'/views/layouts/public'
+end
+
+# dashboard
+get '/', :agent => /(.*)/ do
+  redirect '/dashboard'
+end
+get '/dashboard', :agent => /(.*)/ do
+  erb :'views/dashboard' #, :layout => :'/views/layouts/public'
+end

data/lib/pampa/dispatcher.rb

@@ -0,0 +1,133 @@
+# Pampa Dispatcher
+# Copyright (C) 2022 ExpandedVenture, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+#
+# Authors: Leandro Daniel Sardi (https://github.com/leandrosardi)
+#
+
+require 'pampa'
+
+# parse command line parameters
+PARSER = BlackStack::SimpleCommandLineParser.new(
+    :description => 'This script starts an infinite loop. Each loop will look for a task to perform. Must be a delay between each loop.',
+    :configuration => [{
+        :name=>'delay',
+        :mandatory=>false,
+        :default=>10, # 5 minutes 
+        :description=>'Minimum delay between loops. A minimum of 10 seconds is recommended, in order to don\'t hard the database server. Default is 30 seconds.', 
+        :type=>BlackStack::SimpleCommandLineParser::INT,
+    }, {
+        :name=>'config',
+        :mandatory=>false,
+        :default=>'config.rb', 
+        :description=>'Configuration file. Default: config.', 
+        :type=>BlackStack::SimpleCommandLineParser::STRING,
+    }, {
+        :name=>'db',
+        :mandatory=>false,
+        :default=>'postgres', 
+        :description=>'Database driver. Supported values: postgres, crdb. Default: postgres.', 
+        :type=>BlackStack::SimpleCommandLineParser::STRING,
+    }, {
+        :name=>'log',
+        :mandatory=>false,
+        :default=>true, 
+        :description=>'If write log in the file ./dispatcher.log or not. Default: "yes"', 
+        :type=>BlackStack::SimpleCommandLineParser::BOOL,
+    }]
+)
+
+# create logger
+l = PARSER.value('log') ? BlackStack::LocalLogger.new('dispatcher.log') : BlackStack::BaseLogger.new(nil)
+
+# assign logger to pampa
+BlackStack::Pampa.set_logger(l)
+
+# load config file
+l.logs "Loading #{PARSER.value('config').to_s.blue}... "
+require PARSER.value('config')
+l.logf 'done'.green
+
+l.logs 'Connecting to database... '
+if PARSER.value('db') == 'postgres'
+    DB = BlackStack::PostgreSQL::connect
+elsif PARSER.value('db') == 'crdb'
+    DB = BlackStack::CockroachDB::connect
+else
+    raise 'Unknown database driver.'
+end
+l.logf 'done'.green
+
+# loop
+begin    
+    while true
+        # get the start loop time
+        l.logs 'Starting loop... '
+        start = Time.now()
+        l.logf 'done'.green        
+
+        begin
+            # assign workers to each job
+            l.logs 'Stretching clusters... '
+            BlackStack::Pampa.stretch
+            l.logf 'done'.green
+
+            # relaunch expired tasks
+            l.logs 'Relaunching expired tasks... '
+            BlackStack::Pampa.relaunch
+            l.logf 'done'.green
+
+            # dispatch tasks to each worker
+            l.logs 'Dispatching tasks to workers... '
+            BlackStack::Pampa.dispatch
+            l.logf 'done'.green
+            
+        # note: this catches the CTRL+C signal.
+        # note: this catches the `kill` command, ONLY if it has not the `-9` option.
+        rescue SignalException, SystemExit, Interrupt => e                    
+            l.logf 'Bye!'.yellow
+            raise e
+        rescue => e
+            l.logf "Error: #{e.to_console}".red                
+        end
+        
+        # release resource
+        l.logs 'Releasing resources... '
+        GC.start
+        DB.disconnect
+        l.logf 'done'.green
+        
+        # get the end loop time
+        l.logs 'Ending loop... '
+        finish = Time.now()
+        l.logf 'done'.green
+                
+        # get different in seconds between start and finish
+        # if diff > 30 seconds
+        l.logs 'Calculating loop duration... '
+        diff = finish - start
+        l.logf 'done'.green + " (#{diff.to_s.blue})"
+
+        if diff < PARSER.value('delay')
+            # sleep for 30 seconds
+            n = PARSER.value('delay')-diff
+                    
+            l.logs 'Sleeping for '+n.to_label+' seconds... '
+            sleep n
+            l.logf 'done'.green
+        else
+            l.log 'No sleeping. The loop took '+diff.to_label+' seconds.'
+        end
+    end # while true
+rescue SignalException, SystemExit, Interrupt
+    # note: this catches the CTRL+C signal.
+    # note: this catches the `kill` command, ONLY if it has not the `-9` option.
+    l.logf 'Process Interrumpted.'.yellow
+    l.log 'Bye!'.yellow
+rescue => e
+    l.logf "Fatal Error: #{e.to_console}".red
+rescue 
+    l.logf 'Unknown Fatal Error.'.red
+end # begin

data/lib/pampa/worker.rb

@@ -0,0 +1,172 @@
+# MySaaS - Pampa Worker
+# Copyright (C) 2022 ExpandedVenture, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+#
+# Authors: Leandro Daniel Sardi (https://github.com/leandrosardi)
+#
+
+# load gem and connect database
+require 'pampa'
+
+# parse command line parameters
+PARSER = BlackStack::SimpleCommandLineParser.new(
+    :description => 'This script starts an infinite loop. Each loop will look for a task to perform. Must be a delay between each loop.',
+    :configuration => [{
+        :name=>'delay',
+        :mandatory=>false,
+        :default=>10, 
+        :description=>'Minimum delay between loops. A minimum of 10 seconds is recommended, in order to don\'t hard the database server. Default is 30 seconds.', 
+        :type=>BlackStack::SimpleCommandLineParser::INT,
+    }, {
+        :name=>'config',
+        :mandatory=>false,
+        :default=>'config.rb', 
+        :description=>'Configuration file. Default: config.', 
+        :type=>BlackStack::SimpleCommandLineParser::STRING,
+    }, {
+        :name=>'id', 
+        :mandatory=>true, 
+        :description=>'Write here a unique identifier for the worker.', 
+        :type=>BlackStack::SimpleCommandLineParser::STRING,
+    }, {
+        :name=>'db',
+        :mandatory=>false,
+        :default=>'postgres', 
+        :description=>'Database driver. Values: postgres, crdb. Default: postgres.', 
+        :type=>BlackStack::SimpleCommandLineParser::STRING,
+    }, {
+        :name=>'log',
+        :mandatory=>false,
+        :default=>true, 
+        :description=>'If write log in the file ./worker.#{id}.log or not. Default: "yes"', 
+        :type=>BlackStack::SimpleCommandLineParser::BOOL,
+    }]
+)
+
+# create logger
+l = PARSER.value('log') ? BlackStack::LocalLogger.new("worker.#{PARSER.value('id')}.log") : BlackStack::BaseLogger.new(nil)
+
+# assign logger to pampa
+BlackStack::Pampa.set_logger(l)
+
+# load config file
+l.logs "Loading #{PARSER.value('config').to_s.blue}... "
+require PARSER.value('config')
+l.logf 'done'.green
+
+l.logs 'Connecting to database... '
+if PARSER.value('db') == 'postgres'
+    DB = BlackStack::PostgreSQL::connect
+elsif PARSER.value('db') == 'crdb'
+    DB = BlackStack::CockroachDB::connect
+else
+    raise 'Unknown database driver.'
+end
+l.logf 'done'.green
+
+begin    
+    # getting the worker object
+    l.logs 'Getting worker '+PARSER.value('id').blue+'... '
+    worker = BlackStack::Pampa.workers.select { |w| w.id == PARSER.value('id') }.first
+    raise 'Worker '+PARSER.value('id')+' not found.' if worker.nil?
+    l.logf 'done'.green
+
+    # start the loop
+    while true
+        # get the start loop time
+        l.logs 'Starting loop... '
+        start = Time.now()
+        l.done        
+
+        begin
+            l.log ''
+            l.logs 'Starting cycle... '
+
+            BlackStack::Pampa.jobs.each { |job|
+                task = nil
+                begin
+                    l.logs 'Processing job '+job.name.blue+'... '
+                    tasks = job.occupied_slots(worker)
+                    l.logf tasks.size.to_s+' tasks in queue.'
+
+                    tasks.each { |t|
+                        task = t
+                        
+                        l.logs 'Flag task '+job.name.blue+'.'+task[job.field_primary_key.to_sym].to_s.blue+' started... '
+                        job.start(task)
+                        l.logf 'done'.green
+
+                        l.logs 'Processing task '+task[job.field_primary_key.to_sym].to_s.blue+'... '
+                        job.processing_function.call(task, l, job, worker)
+                        l.logf 'done'.green
+
+                        l.logs 'Flag task '+job.name.blue+'.'+task[job.field_primary_key.to_sym].to_s.blue+' finished... '
+                        job.finish(task)
+                        l.logf 'done'.green
+                    }    
+                # note: this catches the CTRL+C signal.
+                # note: this catches the `kill` command, ONLY if it has not the `-9` option.
+                rescue SignalException, SystemExit, Interrupt => e
+                    l.logs 'Flag task '+job.name.blue+'.'+task[job.field_primary_key.to_sym].to_s.blue+' interrumpted... '
+                    job.finish(task, e)
+                    l.logf 'done'.green
+                        
+                    l.logf 'Bye!'.yellow
+
+                    raise e
+
+                rescue => e
+                    l.logs 'Flag task '+job.name.blue+'.'+task[job.field_primary_key.to_sym].to_s.blue+' failed... '
+                    job.finish(task, e)
+                    l.logf 'done'.green
+
+                    l.logf "Error: #{e.to_console}".red                
+                end        
+            }
+
+            l.done
+
+        rescue => e
+            l.logf 'Error: '+e.message
+        end
+
+        # release resource
+        l.logs 'Releasing resources... '
+        GC.start
+        DB.disconnect
+        l.logf 'done'.green
+
+        # get the end loop time
+        l.logs 'Ending loop... '
+        finish = Time.now()
+        l.logf 'done'.green
+            
+        # get different in seconds between start and finish
+        # if diff > 30 seconds
+        l.logs 'Calculating loop duration... '
+        diff = finish - start
+        l.logf 'done'.green + " (#{diff.to_s.blue})"
+
+        if diff < PARSER.value('delay')
+            # sleep for 30 seconds
+            n = PARSER.value('delay')-diff
+                
+            l.logs 'Sleeping for '+n.to_label+' seconds... '
+            sleep n
+            l.logf 'done'.green
+        else
+            l.log 'No sleeping. The loop took '+diff.to_label.blue+' seconds.'
+        end
+    end # while true
+rescue SignalException, SystemExit, Interrupt
+    # note: this catches the CTRL+C signal.
+    # note: this catches the `kill` command, ONLY if it has not the `-9` option.
+    l.logf 'Process Interrumpted.'.yellow
+    l.log 'Bye!'.yellow
+rescue => e
+    l.logf "Fatal Error: #{e.to_console}".red
+rescue 
+    l.logf 'Unknown Fatal Error.'.red
+end