pampa_workers 1.1.1

data/lib/myremoteprocess.rb

@@ -0,0 +1,138 @@
+module BlackStack
+
+  # no maneja conexion a la base de datos.
+  # ejecuta un loop mientras el metodo canRun? retorne true.
+  class MyRemoteProcess < BlackStack::MyChildProcess
+    # 
+    attr_accessor :worker
+      
+    # update worker configuration in the division
+    def updateWorker()      
+      # creo un remote worker que manejare en este proceso remote
+      self.worker = BlackStack::RemoteWorker.new
+      # me notifico a la central. obtengo asignacion si ya la tenia
+      # y vuelco la configuracion al remote worker
+      url = "#{BlackStack::Pampa::api_url}/api1.3/pampa/get.json"
+      res = BlackStack::Netting::call_post(url, {
+        'api_key' => BlackStack::Pampa::api_key, 
+        'name' => self.fullWorkerName }.merge( BlackStack::RemoteHost.new.poll )
+      )
+      parsed = JSON.parse(res.body)
+      if (parsed['status'] != BlackStack::Netting::SUCCESS)
+        raise parsed['status'].to_s
+      else  
+        self.worker.id                  = parsed['id']
+        self.worker.assigned_process    = parsed['assigned_process']
+        self.worker.id_client           = parsed['id_client']
+        self.worker.id_division         = parsed['id_division']
+        self.worker.division_name       = parsed['division_name']
+        self.worker.ws_url              = parsed['ws_url']
+        self.worker.ws_port             = parsed['ws_port']
+        self.worker.division            = BlackStack::RemoteDivision.new
+        self.worker.division.name       = parsed['division_name']
+      end
+      # llamo al metodo de la clase padre que reporta la configuracion a
+      # la division del worker
+      self.set(parsed['assigned_process'], parsed['id_client'])
+    end
+  
+    # 
+    def run()
+        super
+  
+        # creo el objeto logger
+        self.logger = RemoteLogger.new(
+          "#{self.fullWorkerName}.log",
+          BlackStack::Pampa::api_protocol, 
+          BlackStack::Pampa::api_domain, 
+          BlackStack::Pampa::api_port, 
+          BlackStack::Pampa::api_key,
+          self.id_client # ID of the client that has this thread assigned
+        )
+  
+        logger.log "Remote process is alive!"
+  
+        # actualiza parametros de la central
+        logger.logs "Update from central (1-remote)... "
+        self.get
+        logger.done
+  
+        # actualizo los datos de este worker (parent process)
+        logger.logs "Update worker (1-remote)... "
+        self.updateWorker
+        logger.done
+
+        # actualizo los datos de este worker (parent process)
+        logger.logs "Switch logger id_client (log folder may change)... "
+        self.logger.id_client = self.id_client
+        logger.done
+  
+        while (self.canRun?)
+  
+          # reseteo en contador nested del logger
+          self.logger.reset()
+  
+          # announcing my in the log
+          logger.log "Going to Run Remote"
+          logger.log "Process: #{self.assigned_process.to_s}."
+          logger.log "Client: #{(self.id_client.to_s.size==0)? 'n/a' : self.id_client.to_s}."
+  
+          # obtengo la hora de inicio
+          start_time = Time.now
+  
+          begin
+            # libero recursos
+            logger.logs "Release resources... "
+            GC.start
+            #DB.disconnect
+            logger.done
+  
+            # envia senial a la central.
+            # si tiene asignada una division, envia senial a la division.          
+            logger.logs "Ping... "
+            self.ping()
+            logger.done
+  
+            # envia senial a la central.
+            # si tiene asignada una division, envia senial a la division.          
+            logger.logs "Notify to Division... "
+            self.notify()
+            logger.done
+  
+            # corro el procesamiento
+            self.process(ARGV)
+  
+          rescue => e
+            puts ""
+            logger.log "Remote Process Error: " + e.to_s + "\r\n" + e.backtrace.join("\r\n").to_s      
+          end
+  
+          # actualiza parametros de la central
+          logger.logs "Update from central (2)... "
+          self.get
+          logger.done
+  
+          # actualizo los datos de este worker (parent process)
+          logger.logs "Update worker (2)... "
+          self.updateWorker
+          logger.done
+  
+          # sleep
+          logger.logs "Sleep... "
+          self.doSleep(start_time)
+          logger.done
+  
+          logger.log "-------------------------------------------"
+  
+          GC.start
+          #DB.disconnect
+  
+        end # main while
+  
+        # 
+        logger.log self.whyCantRun()
+        
+    end # run
+  end # class MyRemoteProcess 
+
+end # module BlackStack

data/lib/pampa-local.rb

@@ -0,0 +1,41 @@
+require_relative './division'
+require_relative './params'
+require_relative './worker'
+require_relative './client'
+require_relative './timezone'
+require_relative './user'
+require_relative './login'
+require_relative './role'
+require_relative './userdivision'
+require_relative './userrole'
+
+# funciones auxiliares
+def guid()
+  DB['SELECT NEWID() AS [id]'].map(:id)[0]
+end
+
+def now()
+  # La llamada a GETDATE() desde ODBC no retorna precision de milisegundos, que es necesaria para los registros de log.
+  # La llamada a SYSDATETIME() retorna un valor de alta precision que no es compatible para pegar en los campos del tipo DATATIME.
+  # La linea de abajo obtiene la hora en formato de SYSDATE y le trunca los ultimos digitos para hacer que el valor sea compatible con los campos DATETIME.
+  (DB['SELECT SYSDATETIME() AS [now]'].map(:now)[0]).to_s[0..22]
+end
+
+def diff(unit, t0, t1)
+  # La llamada a GETDATE() desde ODBC no retorna precision de milisegundos, que es necesaria para los registros de log.
+  # La llamada a SYSDATETIME() retorna un valor de alta precision que no es compatible para pegar en los campos del tipo DATATIME.
+  # La linea de abajo obtiene la hora en formato de SYSDATE y le trunca los ultimos digitos para hacer que el valor sea compatible con los campos DATETIME.
+  (DB["SELECT DATEDIFF(#{unit}, '#{t0.to_s}', '#{t1.to_s}') AS [diff]"].map(:diff)[0]).to_i
+end
+
+def before(n) # n minutes
+  DB["SELECT DATEADD(mi, -#{n.to_s}, GETDATE()) AS [now]"].map(:now)[0].to_s[0..22]
+end
+
+def monthsFromNow(n) # n months
+  DB["SELECT DATEADD(mm, +#{n.to_s}, GETDATE()) AS [now]"].map(:now)[0].to_s[0..22]
+end
+
+def daysFromNow(n) # n days
+  DB["SELECT DATEADD(dd, +#{n.to_s}, GETDATE()) AS [now]"].map(:now)[0].to_s[0..22]
+end

data/lib/pampa_workers.rb

@@ -0,0 +1,257 @@
+require 'simple_host_monitoring'
+require 'socket'
+require 'time'
+require 'uri'
+require 'net/http'
+require 'json'
+require 'openssl'
+require 'tiny_tds'
+require 'sequel'
+
+require_relative './baseworker'
+require_relative './basedivision'
+
+require_relative './remoteworker'
+require_relative './remotedivision'
+
+require_relative './myprocess'
+require_relative './mychildprocess'
+require_relative './mylocalprocess'
+require_relative './myparentprocess'
+require_relative './myremoteprocess'
+require_relative './mybotprocess'
+require_relative './mycrawlprocess'
+require_relative './remoteworker'
+require_relative './remotedivision'
+
+module BlackStack
+  
+  module Pampa
+
+    SLEEP_SECONDS = 10
+
+    # 
+    @@division_name = nil
+
+    def self.division_name() 
+      @@division_name
+    end
+
+    #
+    def self.set_division_name(s) 
+      @@division_name = s
+    end
+
+    # Api-key of the client who will be the owner of a process.
+    @@api_key = nil
+    
+    def self.api_key() 
+      @@api_key 
+    end
+    
+    # Protocol, HTTP or HTTPS, of the BlackStack server where this process will be registered.
+    @@api_protocol = nil
+    
+    #
+    def self.api_protocol
+      @@api_protocol
+    end 
+
+    # Domain of the BlackStack server where this process will be registered.
+    @@api_domain = nil
+    
+    #
+    def self.api_domain
+      @@api_domain
+    end
+
+    # Port of the BlackStack server where this process will be registered.
+    @@api_port = nil
+    
+    #
+    def self.api_port
+      @@api_port
+    end
+    
+    # get the full URL of the worker api server
+    def self.api_url()
+      "#{BlackStack::Pampa::api_protocol}://#{BlackStack::Pampa::api_domain}:#{BlackStack::Pampa::api_port}"
+    end
+    
+    # 
+    def self.set_api_key(s)
+      @@api_key = s
+    end
+
+    # 
+    def self.set_api_url(h)
+      @@api_key = h[:api_key]
+      @@api_protocol = h[:api_protocol]
+      @@api_domain = h[:api_domain]
+      @@api_port = h[:api_port]
+    end
+
+    # path where you will store the data of each client
+    @@storage_folder = nil
+    @@storage_sub_folders = []
+
+    #
+    def self.storage_folder()
+      @@storage_folder
+    end
+    def self.storage_sub_folders()
+      @@storage_sub_folders
+    end
+
+    #
+    def self.set_storage_folder(path)
+      @@storage_folder = path
+    end
+    def self.set_storage_sub_folders(a)
+      @@storage_sub_folders = a
+    end
+    
+
+    #
+    # default timezome for any new user
+    #
+    #
+    @@id_timezone_default = nil
+
+    #
+    def self.id_timezone_default()
+      @@id_timezone_default
+    end
+
+    #
+    def self.set_id_timezone_default(id)
+      @@id_timezone_default = id
+    end
+
+    # Array of external IP addresses of the servers where farms are running.
+    # If you have many servers running behind a NAT server, only add the IP 
+    # of the external gateway here. This array is used to know if a worker 
+    # is running inside your farm datacenter, or it is running in the computer 
+    # of someone else.  
+    @@farm_external_ip_addresses = []
+
+    #
+    def self.farm_external_ip_addresses()
+      @@farm_external_ip_addresses
+    end
+
+    #
+    def self.set_farm_external_ip_addresses(a)
+      @@farm_external_ip_addresses = a
+    end
+
+    # 
+    def self.get_guid
+      res = BlackStack::Netting::call_post(
+        "#{self.api_url}/api1.4/get_guid.json",
+        {'api_key' => @@api_key}
+      )
+      parsed = JSON.parse(res.body)
+      parsed['value']        
+    end
+
+    # Central database connection parameters
+    @@db_url = nil
+    @@db_port = nil
+    @@db_name = nil
+    @@db_user = nil
+    @@db_password = nil
+    
+    #
+    def self.db_url
+      @@db_url
+    end
+    
+    #
+    def self.db_port
+      @@db_port
+    end
+    
+    #
+    def self.db_name
+      @@db_name
+    end
+    
+    #
+    def self.db_user
+      @@db_user
+    end
+    
+    #
+    def self.db_password
+      @@db_password
+    end
+
+    # Set connection params to the central database
+    def self.set_db_params(h)
+      @@db_url = h[:db_url]
+      @@db_port = h[:db_port]
+      @@db_name = h[:db_name]
+      @@db_user = h[:db_user]
+      @@db_password = h[:db_password]
+    end
+
+    # TODO: doc me!
+    def self.connection_descriptor()           
+      ret = nil
+      
+      # validar que el formato no sea nulo
+      if (self.division_name.to_s.length == 0)
+        raise "Division name expected."
+      end
+  
+      if (self.division_name == "local") 
+        ret = {
+          :adapter => 'tinytds',
+          :dataserver => BlackStack::Pampa::db_url, # IP or hostname
+          :port => BlackStack::Pampa::db_port, # Required when using other that 1433 (default)
+          :database => BlackStack::Pampa::db_name,
+          :user => BlackStack::Pampa::db_user,
+          :password => BlackStack::Pampa::db_password
+          }      
+      else
+        url = "#{BlackStack::Pampa::api_url}/api1.2/division/get.json"
+        res = BlackStack::Netting::call_post(url, {
+          'api_key' => BlackStack::Pampa::api_key, 
+          'dname' => "#{self.division_name}",
+        })
+        parsed = JSON.parse(res.body)
+        
+        if (parsed["status"] != BlackStack::Netting::SUCCESS)
+          raise "Error getting connection string: #{parsed["status"]}"
+        else
+          wid = parsed["value"]
+          
+          ret = {
+            :adapter => 'tinytds',
+            :dataserver => parsed["db_url"], # IP or hostname
+            :port => parsed["db_port"], # only required if port is different than 1433
+            :database => parsed["db_name"],
+            :user => parsed["db_user"],
+            :password => parsed["db_password"]
+          }
+        end
+      end
+  
+      ret
+    end # connectionDescriptor
+
+    # 
+    def self.db_connection()
+      Sequel.connect(BlackStack::Pampa::connection_descriptor)
+    end
+    
+    #
+    def self.require_db_classes()
+      # You have to load all the Sinatra classes after connect the database.
+      require_relative '../lib/pampa-local.rb'
+    end
+
+  end # module Pampa
+    
+end # module BlackStack

data/lib/params.rb

@@ -0,0 +1,50 @@
+module BlackStack
+  
+  class Params < Sequel::Model(:params)
+    Params.dataset = Params.dataset.disable_insert_output
+    
+    def self.getValue(s)
+      param = Params.where(:name=>s).first
+      if (param == nil)
+        raise "Unknown parameter name (#{s.to_s})"
+      else
+        if param.type == PARAM_TYPE_VARCHAR
+          return param.value_varchar.to_s
+        elsif param.type == PARAM_TYPE_NUMERIC
+          return param.value_numeric.to_i   
+        elsif param.type == PARAM_TYPE_DATETIME
+          return param.value_datetime
+        elsif param.type == PARAM_TYPE_BIT
+          return param.value_bit
+        else
+          raise "Unknown parameter type (#{param.type.to_s})."
+        end # if param.type
+      end # if (param == nil)
+    end # def self.getValue
+  
+    def self.setValue(s, v) # TODO: Testear este metodo
+      param = Params.where(:name=>s).first
+      if (param == nil)
+        raise "Unknown parameter name (#{s.to_s})"
+      else
+        if param.type == PARAM_TYPE_VARCHAR
+          param.value_varchar = v
+          param.save()
+        elsif param.type == PARAM_TYPE_NUMERIC
+          param.value_numeric = v   
+          param.save()
+        elsif param.type == PARAM_TYPE_DATETIME
+          param.value_datetime = v
+          param.save()
+        elsif param.type == PARAM_TYPE_BIT
+          param.value_bit = v
+          param.save()
+        else
+          raise "Unknown parameter type (#{param.type.to_s})."
+        end # if param.type
+      end # if (param == nil)
+    end # def self.getValue
+    
+  end # class
+
+end # module BlackStack