pampa_workers 0.0.38

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fca2d5bf056b264b2630ea17240867025097cff7
+  data.tar.gz: fac05a77c3a324be33f92dfd4f247a0dc7d3a057
+SHA512:
+  metadata.gz: 46793bba1c0d1b06ac04f5004546663cdb2fbe43f8d51dca8c077bc007506fe974ecebce16943f552e0d281f9d9efd2ae1c1251360abf9f0a01dee0528a161f3
+  data.tar.gz: 45705f311c6ca2b7a69fda32d787b84c6a590c6994fb7677d13767a825c493a81607030f9c449dd070f3e21f627d0b88ea311ad9afcbeaa9f57e722ae33103ff

data/lib/basedivision.rb

@@ -0,0 +1,5 @@
+module BlackStack
+  module BaseDivision
+  
+  end # BaseDivision
+end # module BlackStack

data/lib/baseworker.rb

@@ -0,0 +1,5 @@
+module BlackStack
+  module BaseWorker
+    KEEP_ACTIVE_MINUTES = 5 # if the worker didnt send a ping during the last X minutes, it is considered as not-active
+  end # BaseWorker
+end # module BlackStack

data/lib/client.rb

@@ -0,0 +1,245 @@
+require 'invoicing_payments_processing'
+require 'simple_host_monitoring'
+require_relative './user'
+require_relative './role'
+require_relative './timezone'
+
+module BlackStack
+  class Client < Sequel::Model(:client)
+    BlackStack::Client.dataset = BlackStack::Client.dataset.disable_insert_output
+    
+    one_to_many :users, :class=>:'BlackStack::User', :key=>:id_client
+    many_to_one :timezone, :class=>:'BlackStack::Timezone', :key=>:id_timezone
+    many_to_one :billingCountry, :class=>:'BlackStack::LnCountry', :key=>:billing_id_lncountry
+    many_to_one :user_to_contect, :class=>'BlackStack:::User', :key=>:id_user_to_contact
+  
+  
+    # ----------------------------------------------------------------------------------------- 
+    # Arrays
+    # 
+    # 
+    # ----------------------------------------------------------------------------------------- 
+  
+    # returns the workers belong this clients, that have not been deleted
+    def not_deleted_workers()
+      BlackStack::Worker.where(:id_client=>self.id, :delete_time=>nil)
+    end
+  
+    # returns the hosts where this client has not-deleted workers, even if the host is not belong this client
+    def hosts()
+      BlackStack::LocalHost.where(id: self.not_deleted_workers.select(:id_host).all.map(&:id_host))
+    end
+  
+    # returns the hosts belong this client
+    def own_hosts()
+      BlackStack::LocalHost.where(:id_client=>self.id, :delete_time=>nil)
+    end
+  
+    # ----------------------------------------------------------------------------------------- 
+    # Storage:
+    # 
+    # 
+    # ----------------------------------------------------------------------------------------- 
+  
+    # returns the location of the storage for this client
+    def storage_folder
+      "#{BlackStack::Pampa::storage_folder}/#{self.id.to_guid}"
+    end
+  
+    def storage_sub_folder(name)
+      "#{BlackStack::Pampa::storage_folder}/#{self.id.to_guid}/#{name}"
+    end
+  
+    # returns the max allowed KB in the storage for this client
+    def storage_total_kb
+      # TODO: get this parameter from the paid invoces
+      1024*1024 # 1 GB
+    end
+  
+    # returns the max allowed KB in the storage for this client
+    def storage_used_kb
+      path = self.storage_folder
+      fso = WIN32OLE.new('Scripting.FileSystemObject')
+      folder = fso.GetFolder(path)
+      (folder.size.to_f / 1024.to_f) 
+    end
+  
+    # returns the free available KB in the storage for this client
+    def storage_free_kb
+      total = self.storage_total_kb
+      used = self.storage_used_kb
+      total - used
+    end
+    
+    # si el cliente no tiene creado el storage, entonces se lo crea, carpeta por carpeta, ferificando cada una si no existe ya.
+    def create_storage
+      folder = self.storage_folder
+      Dir.mkdir BlackStack::Pampa::storage_folder if Dir[BlackStack::Pampa::storage_folder].size==0
+      if Dir[folder].size==0
+        Dir.mkdir folder
+        
+        BlackStack::Pampa::storage_sub_folders.each { |name|
+          s = "#{folder}/#{name}"
+          Dir.mkdir s if Dir[s].size==0        
+        }      
+      end
+    end
+  
+    # retorna la primera division habilitada a la que pertenezca este cliente
+    def division
+      return self.users.first.division
+    end
+  
+    # ----------------------------------------------------------------------------------------- 
+    # Configuration
+    #
+    #
+    # ----------------------------------------------------------------------------------------- 
+  
+    # retorna true si la 5 variables (billing_address, billing_city, billing_state, billing_zipcode, billing_id_lncountry) tienen un valor destinto a nil o a un string vacio.
+    def hasBillingAddress?
+      if (
+        self.billing_address.to_s.size == 0 || 
+        self.billing_city.to_s.size == 0 || 
+        self.billing_state.to_s.size == 0 || 
+        self.billing_zipcode.to_s.size == 0 ||
+        self.billing_id_lncountry.to_s.size == 0
+      )
+        return false
+      else
+        return true
+      end
+    end
+  
+    # retorna un array de objectos UserRole, asignados a todos los usuarios de este cliente
+    def user_roles
+      a = []
+      self.users.each { |o| 
+        a.concat o.user_roles 
+        # libero recursos
+        GC.start
+        DB.disconnect
+      }
+      a    
+    end
+    
+    # si el cliente no tiene una zona horaria configurada, retorna la zona horaria por defecto
+    # excepciones: 
+    # => "Default timezone not found."
+    def getTimezone()
+      ret = nil   
+      if (self.timezone != nil)
+        ret = self.timezone
+      else
+        ret = BlackStack::Timezone.where(:id=>BlackStack::Pampa::id_timezone_default).first
+        if (ret == nil)
+          raise "Default timezone not found."
+        end
+      end
+      return ret
+    end
+  
+    # llama a la api de postmark preguntando el reseller email configurado para este clietne fue ferificado
+    def checkDomainForSSMVerified()
+      return_message = {}
+      domain = self.domain_for_ssm
+      email = self.from_email_for_ssm
+      id = ''
+      client = ''
+  
+      if domain != nil && email != nil
+        begin
+          client_postmark = Postmark::AccountApiClient.new(POSTMARK_API_TOKEN, secure: true)      
+          client_list = client_postmark.get_signatures()
+  
+          client_list.each do |sign|
+            if sign[:domain] == domain
+              if sign[:email_address] == email
+                id = sign[:id]
+                break
+              end
+            end
+          end
+  
+          if id.to_s.size > 0
+            client = client_postmark.get_sender(id)
+            if !client[:confirmed]
+              self.domain_for_ssm_verified = false
+              self.save
+  
+              return_message[:status] = "No Verified"
+              return_message[:value] = client[:id]
+  
+              return return_message.to_json 
+            else
+              self.domain_for_ssm_verified = true
+              self.save
+  
+              # sincronizo con la central
+              return_message = {}
+  
+              return_message[:status] = "success"
+              return_message[:value] = client[:id]
+  
+              return return_message.to_json 
+            end
+          end
+    
+        rescue Postmark::ApiInputError => e
+          return_message[:status] = e.to_s
+          return return_message.to_json
+          #return e
+        rescue => e
+          return_message[:status] = e.to_s
+          return return_message.to_json
+          #return e
+        end
+      else
+        return_message[:status] = 'error'
+        return_message[:value] = ''
+        return return_message.to_json
+      end # checkDomainForSSMVerified
+    end
+    
+    # retorna true si el cliente esta configurado para usar su propia nombre/email en el envio de notificaciones 
+    def resellerSignature?
+      self.from_name_for_ssm.to_s.size>0 && self.from_email_for_ssm.to_s.size>0 
+    end
+  
+    # retorna true si el cliente esta configurado para usar su propia nombre/email en el envio de notificaciones, y si el email fue verificado en postmark 
+    def resellerSignatureEnabled?
+=begin # TODO: Mover esto a un proceso asincronico
+      # si el cliente esta configurado para usar su propia nombre/email 
+      if self.resellerSignature?
+        # pero el email fue verificado en postmark
+        if self.domain_for_ssm_verified==nil || self.domain_for_ssm_verified!=true
+          # hago la verificacion contra postmark
+          self.checkDomainForSSMVerified   
+        end          
+      end
+=end
+      # return
+      resellerSignature? == true && self.domain_for_ssm_verified==true
+    end
+    
+    # retorna el email configurado y confirmado en PostMark para cuenta reseller, o retorna el email por defecto
+    def resellerSignatureEmail
+      # configuracion de cuenta reseller
+      if self.resellerSignatureEnabled?
+        return self.from_email_for_ssm.to_s
+      else
+        return BlackStack::Params.getValue("notifications.user.email_from")
+      end
+    end
+  
+    # retorna el nombre configurado para cuenta reseller, solo si el email esta confirmado en PostMark; o retorna el email por defecto
+    def resellerSignatureName
+      # configuracion de cuenta reseller
+      if self.resellerSignatureEnabled?
+        return self.from_email_for_ssm.to_s
+      else
+        return BlackStack::Params.getValue("notifications.user.name_from")
+      end
+    end
+  end # class Client
+end # module BlackStack

data/lib/division.rb

@@ -0,0 +1,46 @@
+module BlackStack
+
+  class Division < Sequel::Model(:division)
+    include BlackStack::BaseDivision
+    Division.dataset = Division.dataset.disable_insert_output
+  
+    def home()
+      Division.where(
+        :db_url=>self.db_url, 
+        :db_port=>self.db_port, 
+        :db_user=>self.db_user, 
+        :db_password=>self.db_password, 
+        :db_name=>self.db_name, 
+        :home=>true, 
+        :available=>true
+      ).first
+    end
+  
+    def self.getDefault()
+      q = 
+      "SELECT TOP 1 d.id AS did " +
+      "FROM division d " +
+      "WHERE d.name='#{SIGNUP_DIVISION}' "
+      row = DB[q].first
+      if (row==nil)
+        return nil
+      end
+      return Division.where(:id=>row[:did]).first
+    end
+  
+    # Actualiza el campo stat_name de todas las divisiones que son "gemelas" as la division pasada por parametro.
+    # Ver issue #976.
+    def self.updateStat(division, stat_name, date_time)
+      Division.where(
+        :db_url=>division.db_url,
+        :db_port=>division.db_port,
+        :db_user=>division.db_user,
+        :db_password=>division.db_password,
+      ).each { |d|
+        q = "UPDATE division SET #{stat_name}='#{date_time.to_s}' WHERE id='#{d.id}'"
+      }
+    end
+  
+  end # class
+
+end # module BlackStack

data/lib/login.rb

@@ -0,0 +1,7 @@
+module BlackStack
+  class Login < Sequel::Model(:login)
+    BlackStack::Login.dataset = BlackStack::Login.dataset.disable_insert_output
+    
+    many_to_one :user, :class=>:'BlackStack::User', :key=>:id_user
+  end
+end # module BlackStack

data/lib/mybotprocess.rb

@@ -0,0 +1,348 @@
+module BlackStack
+
+  # clase de base para todos los bots ejecuten acciones con una cuenta de LinkedIn, Facebook, Twitter, etc.
+  class MyBotProcess < BlackStack::MyRemoteProcess
+    attr_accessor :username, :login_verifications, :run_once   
+  
+    # constructor
+    def initialize(
+      the_worker_name, 
+      the_division_name, 
+      the_minimum_enlapsed_seconds=MyProcess::DEFAULT_MINIMUM_ENLAPSED_SECONDS, 
+      the_verify_configuration=true,
+      the_email=nil, 
+      the_password=nil
+    )
+      super(the_worker_name, the_division_name, the_minimum_enlapsed_seconds, the_verify_configuration, the_email, the_password)    
+      self.assigned_process = File.expand_path($0)
+      self.worker_name = "#{the_worker_name}" 
+      self.division_name = the_division_name
+      self.minimum_enlapsed_seconds = the_minimum_enlapsed_seconds
+      
+      # algunas clases como CreateLnUserProcess o RepairLnUserProcess, trabajan unicamente con el username especificado en este atributo, llamando al access point get_lnuser_by_username.
+      # si este atributo es nil, entonces la clase pide un lnuser a la division, llamando al access point get_lnuser.
+      self.username = nil
+      
+      # al correr un proceso sin supervision, el login require verificaciones automaticas que demoran tiempo (account blocingcaptcha, sms pin, bloqueo)
+      # las verificaciones consument tiempo.
+      # si este proceso se corre de forma supevisada, las verificaciones se pueden deshabilitar
+      self.login_verifications = true
+  
+      # al correr sin supervision, el proceso de terminar un un paquete de procesamiento y comenzar con otro, funcionando en un loop infinito.
+      # si este proceso se corre de forma supevisada, se desa correr el procesamiento una unica vez.
+      # cuando se activa este flag, generalmente se setea el atributo self.username tambien. 
+      self.run_once = false
+    end
+  
+    # returns a hash with the parameters of a lnuser
+    # raises an exception if it could not get a lnuser, or if ocurrs any other problem
+    def getLnUserByUsername(username)
+      nTries = 0
+      parsed = nil
+      lnuser = nil # hash
+      bSuccess = false
+      sError = ""
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/login.lnuser/get_lnuser.json"
+          res = BlackStack::Netting::call_post(url, {'api_key' => BlackStack::Pampa::api_key, 'username' => username.encode("UTF-8")})
+          parsed = JSON.parse(res.body)        
+          if (parsed['status']=='success')
+            lnuser = parsed
+            bSuccess = true
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception: " + e2.to_s
+        end
+      end # while
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+  
+      return lnuser
+    end # getLnUser()
+  
+    # returns a hash with the parameters of a lnuser
+    # raises an exception if it could not get a lnuser, or if ocurrs any other problem
+    def getLnUser(workflow_name='incrawl.lnsearchvariation')
+      nTries = 0
+      parsed = nil
+      lnuser = nil # hash
+      bSuccess = false
+      sError = ""
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/#{workflow_name}/get_lnuser.json"
+          res = BlackStack::Netting::call_post(url, {'api_key' => BlackStack::Pampa::api_key, 'name' => self.fullWorkerName})
+          parsed = JSON.parse(res.body)        
+          if (parsed['status']=='success')
+            lnuser = parsed
+            bSuccess = true
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception" + e2.to_s
+        end
+      end # while
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+  
+      return lnuser
+    end # getLnUser()
+  
+    #  
+    def notifyLnUserUrl(id_lnuser, url)
+=begin
+      nTries = 0
+      parsed = nil
+      bSuccess = false
+      sError = ""
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/login.lnuser/notify_url.json"
+          res = BlackStack::Netting::call_post(url,
+            {:api_key => BlackStack::Pampa::api_key,
+            'id_lnuser' => id_lnuser,
+            'url' => url,}
+          )
+          parsed = JSON.parse(res.body)
+          
+          if (parsed['status']=='success')
+            bSuccess = true
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception" + e2.to_s
+        end
+      end # while 
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+=end
+    end # notifyLnUserStatus
+  
+    #  
+    def notifyLnUserStatus(id_lnuser, status, workflow_name='incrawl.lnsearchvariation')
+      nTries = 0
+      parsed = nil
+      bSuccess = false
+      sError = ""
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/#{workflow_name}/notify_lnuser_status.json"
+          res = BlackStack::Netting::call_post(url,
+            {'api_key' => BlackStack::Pampa::api_key, 
+            'id_lnuser' => id_lnuser,
+            'status' => status,}
+          )
+          parsed = JSON.parse(res.body)
+          
+          if (parsed['status']=='success')
+            bSuccess = true
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception" + e2.to_s
+        end
+      end # while 
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+  
+    end # notifyLnUserStatus
+  
+    #  
+    def notifyLnUserActivity(id_lnuser, code, workflow_name='incrawl.lnsearchvariation')
+      nTries = 0
+      parsed = nil
+      bSuccess = false
+      sError = ""
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/#{workflow_name}/notify_lnuser_activity.json"
+          res = BlackStack::Netting::call_post(url,
+            {'api_key' => BlackStack::Pampa::api_key, 
+            'id_lnuser' => id_lnuser,
+            'code' => code,}
+          )
+          parsed = JSON.parse(res.body)
+          
+          if (parsed['status']=='success')
+            bSuccess = true
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception" + e2.to_s
+        end
+      end # while 
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+    end # notifyLnUserStatus
+  
+    # Toma una captura del browser.
+    # Sube un registro a la tabla boterrorlog, con el id del worker, el proceso asinado, y el screenshot. 
+    #
+    # uid: id de un registro en la tabla lnuser.
+    # description: backtrace de la excepcion.
+    #
+    def notifyError(uid, description, oid=nil)
+      # tomo captura de pantalla
+      file = nil
+=begin # TODO: habilitar esto cuando se migre a RestClient en vez de CallPost
+      begin
+        screenshot_filename = "./error.png" # TODO: colocar un nombre unico formado por por el fullname del worker, y la fecha-hora.
+        BrowserFactory.screenshot screenshot_filename
+        file = File.new(screenshot_filename, "rb")
+      rescue => e
+  puts "Screenshot Error: #{e.to_s}"
+        file = nil
+      end
+=end
+  #puts ""
+  #puts "id_worker:#{PROCESS.worker.id}"
+  #puts "worker_name:#{PROCESS.fullWorkerName}"
+  #puts "process:#{PROCESS.worker.assigned_process}"
+  #puts ""
+      # subo el error
+      nTries = 0
+      bSuccess = false
+      parsed = nil
+      sError = ""
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/boterror.json"
+          res = BlackStack::Netting::call_post(url, # TODO: migrar a RestClient para poder hacer file upload
+            'api_key' => BlackStack::Pampa::api_key, 
+            'id_lnuser' => uid, 
+            'id_object' => oid, 
+            'worker_name' => PROCESS.fullWorkerName, 
+            'process' => PROCESS.worker.assigned_process,
+            'description' => description,
+            'screenshot' => file,
+          )
+          parsed = JSON.parse(res.body)
+          if (parsed['status']=='success')
+            bSuccess = true
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception" + e2.to_s
+        end
+      end # while
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+    end
+  
+    #  
+    def isLnUserAvailable(id_lnuser, need_sales_navigator=false, workflow_name='incrawl.lnsearchvariation')
+      nTries = 0
+      parsed = nil
+      bSuccess = false
+      sError = ""
+      ret = false
+      
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/#{workflow_name}/is_lnuser_available.json"
+          res = BlackStack::Netting::call_post(url,
+            {'api_key' => BlackStack::Pampa::api_key, 
+            'id_lnuser' => id_lnuser,
+            'need_sales_navigator' => need_sales_navigator,}
+          )
+          parsed = JSON.parse(res.body)
+          
+          if (parsed['status']=='success')
+            bSuccess = true
+            ret = parsed['value']
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Alghoritm Exception" + e2.to_s + '\r\n' + e2.backtrace.join("\r\n").to_s
+        end
+      end # while 
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+      
+      return ret
+    end # isLnUserAvailable
+  
+    #  
+    def releaseLnUser(id_lnuser, workflow_name='incrawl.lnsearchvariation')
+      nTries = 0
+      parsed = nil
+      bSuccess = false
+      sError = ""
+      ret = false
+      
+      while (nTries < 5 && bSuccess == false)
+        begin
+          nTries = nTries + 1
+          url = "#{BlackStack::Pampa::api_protocol}://#{self.ws_url}:#{self.ws_port}/api1.3/pampa/#{workflow_name}/release_lnuser.json"
+          res = BlackStack::Netting::call_post(url,
+            {'api_key' => BlackStack::Pampa::api_key, 'id_lnuser' => id_lnuser,}
+          )
+          parsed = JSON.parse(res.body)
+          
+          if (parsed['status']=='success')
+            bSuccess = true
+            ret = parsed['value']
+          else
+            sError = parsed['status']
+          end
+        rescue Errno::ECONNREFUSED => e
+          sError = "Errno::ECONNREFUSED:" + e.to_s
+        rescue => e2
+          sError = "Exception" + e2.to_s
+        end
+      end # while 
+  
+      if (bSuccess==false)
+        raise "#{sError}"
+      end
+      
+      return ret
+    end # isLnUserAvailable
+  
+  end # class MyBotProcess
+
+end # module BlackStack

data/lib/mychildprocess.rb

@@ -0,0 +1,9 @@
+module BlackStack
+  
+  # es un proceso sin conexion a base de datos, que itera infinitamente.
+  # en cada iteracion saluda a la central (hello), obtiene parametros (get)
+  class MyChildProcess < BlackStack::MyProcess
+
+  end # MyChildProcess
+
+end # module BlackStack