pampa_workers 0.0.38

data/lib/myremoteprocess.rb

@@ -0,0 +1,128 @@
+module BlackStack
+
+  # no maneja conexion a la base de datos.
+  # ejecuta un loop mientras el metodo canRun? retorne true.
+  class MyRemoteProcess < BlackStack::MyChildProcess
+    # 
+    attr_accessor :worker
+      
+    # update worker configuration in the division
+    def updateWorker()
+      # 
+      self.worker = BlackStack::RemoteWorker.new
+      # me notifico a la central. obtengo asignacion si ya la tenia
+      url = "#{BlackStack::Pampa::api_url}/api1.3/pampa/get.json"
+      res = BlackStack::Netting::call_post(url, {
+        'api_key' => BlackStack::Pampa::api_key, 
+        'name' => self.fullWorkerName }.merge( BlackStack::RemoteHost.new.poll )
+      )
+      parsed = JSON.parse(res.body)
+      if (parsed['status'] != BlackStack::Netting::SUCCESS)
+        raise parsed['status'].to_s
+      else  
+        self.worker.id                 = parsed['id']
+        self.worker.assigned_process   = parsed['assigned_process']
+        self.worker.id_object          = parsed['id_object']
+        self.worker.id_division        = parsed['id_division']
+        self.worker.division_name      = parsed['division_name']
+        self.worker.ws_url             = parsed['ws_url']
+        self.worker.ws_port            = parsed['ws_port']
+        self.worker.division                  = BlackStack::RemoteDivision.new
+        self.worker.division.name             = parsed['division_name']
+      end
+    end
+  
+    # 
+    def run()
+        super
+  
+        # creo el objeto logger
+        self.logger = RemoteLogger.new(
+          "#{self.fullWorkerName}.log",
+          BlackStack::Pampa::api_protocol, 
+          BlackStack::Pampa::api_domain, 
+          BlackStack::Pampa::api_port, 
+          BlackStack::Pampa::api_key
+        )
+  
+        logger.log "Remote process is alive!"
+  
+        # actualiza parametros de la central
+        logger.logs "Update from central (1)... "
+        self.get
+        logger.done
+  
+        # actualizo los datos de este worker (parent process)
+        logger.logs "Update worker (1)... "
+        self.updateWorker
+        logger.done
+  
+        while (self.canRun?)
+  
+          # reseteo en contador nested del logger
+          self.logger.reset()
+  
+          # announcing my in the log
+          logger.log "Going to Run Remote"
+          logger.log "Process: #{self.assigned_process.to_s}."
+          logger.log "Object: #{(self.id_object.to_s.size==0)? 'n/a' : self.id_object.to_s}."
+  
+          # obtengo la hora de inicio
+          start_time = Time.now
+  
+          begin
+            # libero recursos
+            logger.logs "Release resources... "
+            GC.start
+            #DB.disconnect
+            logger.done
+  
+            # envia senial a la central.
+            # si tiene asignada una division, envia senial a la division.          
+            logger.logs "Ping... "
+            self.ping()
+            logger.done
+  
+            # envia senial a la central.
+            # si tiene asignada una division, envia senial a la division.          
+            logger.logs "Notify to Division... "
+            self.notify()
+            logger.done
+  
+            # corro el procesamiento
+            self.process(ARGV)
+  
+          rescue => e
+            puts ""
+            logger.log "Remote Process Error: " + e.to_s + "\r\n" + e.backtrace.join("\r\n").to_s      
+          end
+  
+          # actualiza parametros de la central
+          logger.logs "Update from central (2)... "
+          self.get
+          logger.done
+  
+          # actualizo los datos de este worker (parent process)
+          logger.logs "Update worker (2)... "
+          self.updateWorker
+          logger.done
+  
+          # sleep
+          logger.logs "Sleep... "
+          self.doSleep(start_time)
+          logger.done
+  
+          logger.log "-------------------------------------------"
+  
+          GC.start
+          #DB.disconnect
+  
+        end # main while
+  
+        # 
+        logger.log self.whyCantRun()
+        
+    end # run
+  end # class MyRemoteProcess 
+
+end # module BlackStack

data/lib/pampa-local.rb

@@ -0,0 +1,41 @@
+require_relative './division'
+require_relative './params'
+require_relative './worker'
+require_relative './client'
+require_relative './timezone'
+require_relative './user'
+require_relative './login'
+require_relative './role'
+require_relative './userdivision'
+require_relative './userrole'
+
+# funciones auxiliares
+def guid()
+  DB['SELECT NEWID() AS [id]'].map(:id)[0]
+end
+
+def now()
+  # La llamada a GETDATE() desde ODBC no retorna precision de milisegundos, que es necesaria para los registros de log.
+  # La llamada a SYSDATETIME() retorna un valor de alta precision que no es compatible para pegar en los campos del tipo DATATIME.
+  # La linea de abajo obtiene la hora en formato de SYSDATE y le trunca los ultimos digitos para hacer que el valor sea compatible con los campos DATETIME.
+  (DB['SELECT SYSDATETIME() AS [now]'].map(:now)[0]).to_s[0..22]
+end
+
+def diff(unit, t0, t1)
+  # La llamada a GETDATE() desde ODBC no retorna precision de milisegundos, que es necesaria para los registros de log.
+  # La llamada a SYSDATETIME() retorna un valor de alta precision que no es compatible para pegar en los campos del tipo DATATIME.
+  # La linea de abajo obtiene la hora en formato de SYSDATE y le trunca los ultimos digitos para hacer que el valor sea compatible con los campos DATETIME.
+  (DB["SELECT DATEDIFF(#{unit}, '#{t0.to_s}', '#{t1.to_s}') AS [diff]"].map(:diff)[0]).to_i
+end
+
+def before(n) # n minutes
+  DB["SELECT DATEADD(mi, -#{n.to_s}, GETDATE()) AS [now]"].map(:now).to_s[0]
+end
+
+def monthsFromNow(n) # n months
+  DB["SELECT DATEADD(mm, +#{n.to_s}, GETDATE()) AS [now]"].map(:now).to_s[0]
+end
+
+def daysFromNow(n) # n days
+  DB["SELECT DATEADD(dd, +#{n.to_s}, GETDATE()) AS [now]"].map(:now).to_s[0]
+end

data/lib/pampa.rb

@@ -0,0 +1,261 @@
+require 'blackstack_commons'
+require 'simple_command_line_parser'
+require 'simple_cloud_logging'
+require 'simple_host_monitoring'
+require 'socket'
+require 'time'
+require 'uri'
+require 'net/http'
+require 'json'
+require 'openssl'
+require 'tiny_tds'
+require 'sequel'
+
+#require './lib/base'
+#require './config.rb'
+
+require_relative './baseworker'
+require_relative './basedivision'
+
+require_relative './remoteworker'
+require_relative './remotedivision'
+
+require_relative './myprocess'
+require_relative './mychildprocess'
+require_relative './mylocalprocess'
+require_relative './myparentprocess'
+require_relative './myremoteprocess'
+require_relative './mybotprocess'
+require_relative './mycrawlprocess'
+require_relative './remoteworker'
+require_relative './remotedivision'
+
+module BlackStack
+  
+  module Pampa
+
+    # 
+    @@division_name = nil
+
+    def self.division_name() 
+      @@division_name
+    end
+
+    #
+    def self.set_division_name(s) 
+      @@division_name = s
+    end
+
+    # Api-key of the client who will be the owner of a process.
+    @@api_key = nil
+    
+    def self.api_key() 
+      @@api_key 
+    end
+    
+    # Protocol, HTTP or HTTPS, of the BlackStack server where this process will be registered.
+    @@api_protocol = nil
+    
+    #
+    def self.api_protocol
+      @@api_protocol
+    end 
+
+    # Domain of the BlackStack server where this process will be registered.
+    @@api_domain = nil
+    
+    #
+    def self.api_domain
+      @@api_domain
+    end
+
+    # Port of the BlackStack server where this process will be registered.
+    @@api_port = nil
+    
+    #
+    def self.api_port
+      @@api_port
+    end
+    
+    # get the full URL of the worker api server
+    def self.api_url()
+      "#{BlackStack::Pampa::api_protocol}://#{BlackStack::Pampa::api_domain}:#{BlackStack::Pampa::api_port}"
+    end
+    
+    # 
+    def self.set_api_key(s)
+      @@api_key = s
+    end
+
+    # 
+    def self.set_api_url(h)
+      @@api_key = h[:api_key]
+      @@api_protocol = h[:api_protocol]
+      @@api_domain = h[:api_domain]
+      @@api_port = h[:api_port]
+    end
+
+    # path where you will store the data of each client
+    @@storage_folder = nil
+    @@storage_sub_folders = []
+
+    #
+    def self.storage_folder()
+      @@storage_folder
+    end
+    def self.storage_sub_folders(a)
+      @@storage_sub_folder = a
+    end
+
+    #
+    def self.set_storage_folder(path)
+      @@storage_folder = path
+    end
+    def self.set_storage_sub_folders(a)
+      @@storage_sub_folders = a
+    end
+    
+
+    #
+    # default timezome for any new user
+    #
+    #
+    @@id_timezone_default = nil
+
+    #
+    def self.id_timezone_default()
+      @@id_timezone_default
+    end
+
+    #
+    def self.set_id_timezone_default(id)
+      @@id_timezone_default = id
+    end
+
+    # Array of external IP addresses of the servers where farms are running.
+    # If you have many servers running behind a NAT server, only add the IP 
+    # of the external gateway here. This array is used to know if a worker 
+    # is running inside your farm datacenter, or it is running in the computer 
+    # of someone else.  
+    @@farm_external_ip_addresses = []
+
+    #
+    def self.farm_external_ip_addresses()
+      @@farm_external_ip_addresses
+    end
+
+    #
+    def self.set_farm_external_ip_addresses(a)
+      @@farm_external_ip_addresses = a
+    end
+
+    # 
+    def self.get_guid
+      res = BlackStack::Netting::call_post(
+        "#{self.api_url}/api1.4/get_guid.json",
+        {'api_key' => @@api_key}
+      )
+      parsed = JSON.parse(res.body)
+      parsed['value']        
+    end
+
+    # Central database connection parameters
+    @@db_url = nil
+    @@db_port = nil
+    @@db_name = nil
+    @@db_user = nil
+    @@db_password = nil
+    
+    #
+    def self.db_url
+      @@db_url
+    end
+    
+    #
+    def self.db_port
+      @@db_port
+    end
+    
+    #
+    def self.db_name
+      @@db_name
+    end
+    
+    #
+    def self.db_user
+      @@db_user
+    end
+    
+    #
+    def self.db_password
+      @@db_password
+    end
+
+    # Set connection params to the central database
+    def self.set_db_params(h)
+      @@db_url = h[:db_url]
+      @@db_port = h[:db_port]
+      @@db_name = h[:db_name]
+      @@db_user = h[:db_user]
+      @@db_password = h[:db_password]
+    end
+
+    # TODO: doc me!
+    def self.connection_descriptor()           
+      ret = nil
+      
+      # validar que el formato no sea nulo
+      if (self.division_name.to_s.length == 0)
+        raise "Division name expected."
+      end
+  
+      if (self.division_name == "local") 
+        ret = {
+          :adapter => 'tinytds',
+          :dataserver => BlackStack::Pampa::db_url, # IP or hostname
+          :port => BlackStack::Pampa::db_port, # Required when using other that 1433 (default)
+          :database => BlackStack::Pampa::db_name,
+          :user => BlackStack::Pampa::db_user,
+          :password => BlackStack::Pampa::db_password
+          }      
+      else
+        url = "#{BlackStack::Pampa::api_url}/api1.2/division/get.json"
+        res = BlackStack::Netting::call_post(url, {
+          'api_key' => BlackStack::Pampa::api_key, 
+          'dname' => "#{self.division_name}",
+        })
+        parsed = JSON.parse(res.body)
+        
+        if (parsed["status"] != BlackStack::Netting::SUCCESS)
+          raise "Error getting connection string: #{parsed["status"]}"
+        else
+          wid = parsed["value"]
+          
+          ret = {
+            :adapter => 'tinytds',
+            :dataserver => parsed["db_url"], # IP or hostname
+            :port => parsed["db_port"], # only required if port is different than 1433
+            :database => parsed["db_name"],
+            :user => parsed["db_user"],
+            :password => parsed["db_password"]
+          }
+        end
+      end
+  
+      ret
+    end # connectionDescriptor
+
+    # 
+    def self.db_connection()
+      Sequel.connect(BlackStack::Pampa::connection_descriptor)
+    end
+    
+    #
+    def self.require_db_classes()
+      # You have to load all the Sinatra classes after connect the database.
+      require_relative '../lib/pampa-local.rb'
+    end
+
+  end # module Pampa
+    
+end # module BlackStack

data/lib/params.rb

@@ -0,0 +1,50 @@
+module BlackStack
+  
+  class Params < Sequel::Model(:params)
+    Params.dataset = Params.dataset.disable_insert_output
+    
+    def self.getValue(s)
+      param = Params.where(:name=>s).first
+      if (param == nil)
+        raise "Unknown parameter name (#{s.to_s})"
+      else
+        if param.type == PARAM_TYPE_VARCHAR
+          return param.value_varchar.to_s
+        elsif param.type == PARAM_TYPE_NUMERIC
+          return param.value_numeric.to_i   
+        elsif param.type == PARAM_TYPE_DATETIME
+          return param.value_datetime
+        elsif param.type == PARAM_TYPE_BIT
+          return param.value_bit
+        else
+          raise "Unknown parameter type (#{param.type.to_s})."
+        end # if param.type
+      end # if (param == nil)
+    end # def self.getValue
+  
+    def self.setValue(s, v) # TODO: Testear este metodo
+      param = Params.where(:name=>s).first
+      if (param == nil)
+        raise "Unknown parameter name (#{s.to_s})"
+      else
+        if param.type == PARAM_TYPE_VARCHAR
+          param.value_varchar = v
+          param.save()
+        elsif param.type == PARAM_TYPE_NUMERIC
+          param.value_numeric = v   
+          param.save()
+        elsif param.type == PARAM_TYPE_DATETIME
+          param.value_datetime = v
+          param.save()
+        elsif param.type == PARAM_TYPE_BIT
+          param.value_bit = v
+          param.save()
+        else
+          raise "Unknown parameter type (#{param.type.to_s})."
+        end # if param.type
+      end # if (param == nil)
+    end # def self.getValue
+    
+  end # class
+
+end # module BlackStack

data/lib/remotedivision.rb

@@ -0,0 +1,8 @@
+module BlackStack
+
+  class RemoteDivision
+      attr_accessor :name
+      include BlackStack::BaseDivision
+  end # RemoteDivision
+
+end # module BlackStack

data/lib/remoteworker.rb

@@ -0,0 +1,8 @@
+module BlackStack
+
+  class RemoteWorker
+      attr_accessor :id, :process, :last_ping_time, :name, :active, :id_division, :assigned_process, :id_object, :division_name, :ws_url, :ws_port, :division
+      include BlackStack::BaseWorker
+  end # Remote Worker
+  
+end # module BlackStack

data/lib/role.rb

@@ -0,0 +1,8 @@
+module BlackStack
+  class Role < Sequel::Model(:role)
+    BlackStack::Role.dataset = BlackStack::Role.dataset.disable_insert_output
+  
+    ROLE_PRISMA_USER = "prisma.user"
+  
+  end
+end # module BlackStack

data/lib/timezone.rb

@@ -0,0 +1,151 @@
+module BlackStack
+  class Timezone < Sequel::Model(:timezone)
+    BlackStack::Timezone.dataset = BlackStack::Timezone.dataset.disable_insert_output
+  
+    # Recibe un string con formato "+HH:MM".
+    # Retorna la cantidad de horas como un numero real, con decimales.
+    def self.descToFloat(s)
+      sign = s[0]
+      if (sign=="+")
+        n = 1.0
+      else
+        n = -1.0
+      end
+      hours = s[1..2]
+      minutes = s[4..5]
+      int_part = hours.to_f
+      dec_part = minutes.to_f / 60.00
+      ret = n*(int_part + dec_part)    
+      return ret.to_f 
+    end
+  
+    # Recibe la cantidad de horas como un numero real, con decimales.
+    # Retorna un string con formato "+HH:MM".
+    def self.floatToDesc(x)
+      int_part = x.to_i
+      dec_part = (x-int_part.to_f).round(2).abs
+      hours = int_part
+      minutes = (dec_part * 60.00).to_i    
+      if (hours<0)
+        desc = "%03d" % hours 
+      else
+        desc = "+" + "%02d" % hours
+      end
+      desc += ":" + "%02d" % minutes
+      return desc
+    end
+  
+    # Example: '-03:00'
+    # Example: ' 08:00'
+    def self.getDatabaseOffset()
+      dbtime = DB["select SYSDATETIMEOFFSET() AS o"].first[:o].to_s
+  #puts ""
+  #puts ""
+  #puts "dbtime:#{dbtime.to_s}:."
+  #puts ""
+  #puts ""
+      ret = "#{dbtime[-5..-3]}:#{dbtime[-2..-1]}" 
+      ret[0] = '+' if ret[0]!='-'
+  #puts ""
+  #puts ""
+  #puts "dbtime:#{ret.to_s}:."
+  #puts ""
+  #puts ""
+      return ret
+    end
+  
+    # Convierte el atributo offset en un string con formato de hora.
+    # Ejemplos: 5.0 => 05:00, 5.5 => 05:30, -5.75 => -05:45   
+    def offsetDescription()
+      return BlackStack::Timezone.floatToDesc(self.offset.to_f)
+    end
+    
+    # Convierte una hora almacenada en el servidor de bases de datos, a la hora en esta zona horaria.
+    # Recibe un string con formato "+HH:MM".
+    # Retorna un string con formato "+HH:MM".
+    def convertFromThisTimeZoneToDatabaseTime(s_time_in_client_timezone)
+      f_time_in_client_timezone = BlackStack::Timezone.descToFloat(s_time_in_client_timezone)
+      s = BlackStack::Timezone.getDatabaseOffset()
+      x = BlackStack::Timezone.descToFloat(s) # database offset
+      y = self.offset # client offset
+      z = y - x 
+      return BlackStack::Timezone.floatToDesc((f_time_in_client_timezone.to_f + z.to_f).modulo(24))
+    end
+  
+    # Convierte una hora almacenada en el servidor de bases de datos, a la hora en esta zona horaria.
+    # Recibe un string con formato "+HH:MM".
+    # Retorna un string con formato "+HH:MM".
+    def convertFromDatabaseTimeToThisTimeZone(s_time_in_database)
+      f_time_in_database = BlackStack::Timezone.descToFloat(s_time_in_database)
+      s = BlackStack::Timezone.getDatabaseOffset()
+      x = BlackStack::Timezone.descToFloat(s) # database offset
+      y = self.offset # client offset
+      z = y - x 
+      return BlackStack::Timezone.floatToDesc((f_time_in_database.to_f - z.to_f).modulo(24))
+    end
+  
+    # Convierte una fecha-hora almacenada en el servidor de bases de datos, a la hora en esta zona horaria.
+    # Recibe un string con formato "YYYY-MM-DD HH:MM:SS".
+    # Retorna un string con formato "YYYY-MM-DD HH:MM:SS".
+    def convertFromDatabaseDateTimeToThisTimeZone(s_datetime_in_database)
+      s = BlackStack::Timezone.getDatabaseOffset() # Example: '-03:00'
+  #puts ""
+  #puts ""
+  #puts "s:#{s.to_s}:."
+  #puts ""
+  #puts ""
+      x = BlackStack::Timezone.descToFloat(s) # database offset. Number of hours, with decimals
+  #puts ""
+  #puts ""
+  #puts "x:#{x.to_s}:."
+  #puts ""
+  #puts ""
+      y = self.offset # client offset
+  #puts ""
+  #puts ""
+  #puts "y:#{y.to_s}:."
+  #puts ""
+  #puts ""
+      z = y - x 
+  #puts ""
+  #puts ""
+  #puts "z:#{z.to_s}:."
+  #puts ""
+  #puts ""
+      ret = DB["SELECT DATEADD(HH, #{z.to_s}, '#{s_datetime_in_database}') AS o"].first[:o].to_s[0..18]
+  #puts ""
+  #puts ""
+  #puts "convertFromDatabaseDateTimeToThisTimeZone:#{ret}:."
+  #puts ""
+  #puts ""
+  
+      return ret
+    end
+    
+    # 
+    def get_now()
+      datetime1 = DB["SELECT GETDATE() AS o"].first[:o].to_s[0..18]
+      datetime2 = self.convertFromDatabaseDateTimeToThisTimeZone(datetime1)
+  #puts ""
+  #puts ""
+  #puts "timezone:#{self.large_description}:."
+  #puts "datetime1:#{datetime1}:."
+  #puts "datetime2:#{datetime2}:."
+  #puts ""
+  #puts ""
+      yy = datetime2[0..3] # yyyy-mm-dd hh:mi:ss
+      mm = datetime2[5..6]
+      dd = datetime2[8..9]
+      hh = datetime2[11..12]
+      mi = datetime2[14..15]
+      ss = datetime2[17..18]
+  #puts ""
+  #puts ""
+  #puts "get_now:#{yy}-#{mm}-#{dd} #{hh}:#{mi}:#{ss}:."
+  #puts ""
+  #puts ""
+      return Time.new(yy,mm,dd,hh,mi,ss)
+    end
+    
+  end
+end # module BlackStack

data/lib/user.rb

@@ -0,0 +1,25 @@
+module BlackStack
+  class User < Sequel::Model(:user)
+    BlackStack::User.dataset = BlackStack::User.dataset.disable_insert_output
+    many_to_one :client, :class=>:'BlackStack::Client', :key=>:id_client
+    one_to_many :user_roles, :class=>:'BlackStack::UserRole', :key=>:id_user
+  
+  # TODO: agregar todos los arrays de un usuario
+  # => one_to_many :searches, :class=>:Search, :key=>:id_user
+  # => one_to_many :emlist, :class=>:EmList, :key=>:id_user
+  # => one_to_many :emlist, :class=>:EmList, :key=>:id_user
+  # => etc.
+  
+    # retorna la primera division habilitada a la que pertenezca este usuario
+    def division
+      row = DB[
+      "SELECT d.id " +
+      "FROM division d WITH (NOLOCK) " +
+      "JOIN user_division ud WITH (NOLOCK) ON (d.id=ud.id_division AND ud.id_user='#{self.id}') " +
+      "WHERE ISNULL(d.available,0) = 1 "
+      ].first
+      return nil if row.nil?
+      return BlackStack::Division.where(:id=>row[:id]).first if !row.nil?
+    end  
+  end # class User
+end # module BlackStack

data/lib/userdivision.rb

@@ -0,0 +1,5 @@
+module BlackStack
+  class UserDivision < Sequel::Model(:user_division)
+    BlackStack::UserDivision.dataset = BlackStack::UserDivision.dataset.disable_insert_output
+  end
+end # module BlackStack

data/lib/userrole.rb

@@ -0,0 +1,8 @@
+module BlackStack
+  class UserRole < Sequel::Model(:user_role)
+    BlackStack::UserRole.dataset = BlackStack::UserRole.dataset.disable_insert_output
+    many_to_one :user, :class=>:'BlackStack::User', :key=>:id_user
+    many_to_one :role, :class=>:'BlackStack::Role', :key=>:id_role
+  
+  end
+end # module BlackStack