pampa_workers 0.0.38

data/lib/worker.rb

@@ -0,0 +1,234 @@
+module BlackStack
+
+  # 
+  class WorkerJob < Sequel::Model(:workerjob)
+     
+  end
+
+  # 
+  class Worker < Sequel::Model(:worker) 
+    include BlackStack::BaseWorker
+    BlackStack::Worker.dataset = BlackStack::Worker.dataset.disable_insert_output
+    many_to_one :division, :class=>:'BlackStack::Division', :key=>:id_division
+    many_to_one :user, :class=>:'BlackStack::User', :key=>:id_user
+=begin
+    # deprecated
+    # Actualiza la la lista de workers que estan asignados esta division
+    def self.updateAllFromCentral()
+      uri = URI("#{WORKER_API_SERVER_URL}/api1.3/pampa/get_all.json")
+      res = Net::HTTP.post_form(uri, {:api_key => BlackStack::Pampa::api_key,})
+      parsed = JSON.parse(res.body)
+      if (parsed['status'] != "success")
+        raise parsed['status'].to_s
+      else
+        parsed['workers'].each { |worker|
+          if ( worker['division_name']!=DATABASE )
+            q = "UPDATE worker SET active=0, division_name='#{worker['division_name']}' WHERE name='#{worker['name']}'"
+            DB.execute(q)
+          else # worker['division_name']==DIVISION_NAME   
+            
+            worker = BlackStack::Worker.where(:name=>worker['name']).first
+            if (worker==nil)
+              worker = BlackStack::Worker.new()
+              worker.id = worker['id']
+              worker.name = worker['name'].to_s
+              worker.last_ping_time = now() # esta fecha es actualiada por el mismo worker, para indicar que esta vivo y trabajando
+              worker.id_division = worker['id_division']
+              worker.process = worker['assigned_process']
+              worker.assigned_process = worker['assigned_process'] 
+              worker.id_object = worker['id_object']
+              worker.division_name = worker['division_name']
+              worker.save()
+            else
+              #puts "update" ?            
+            end
+            
+            DB.execute("UPDATE worker SET active=1 WHERE name='#{worker['name'].to_s}'")
+  
+            if (worker['id_division'] != nil)
+              DB.execute("UPDATE worker SET id_division='#{worker['id_division'].to_s}' WHERE name='#{worker['name'].to_s}'")
+            end
+  
+            if (worker['assigned_process'] != nil)
+              DB.execute("UPDATE worker SET process='#{worker['assigned_process'].to_s}', assigned_process='#{worker['assigned_process'].to_s}' WHERE name='#{worker['name'].to_s}'")
+            end
+  
+            if (worker['id_object'] != nil)
+              DB.execute("UPDATE worker SET id_object='#{worker['id_object'].to_s}' WHERE name='#{worker['name'].to_s}'")
+            end
+  
+            if (worker['division_name'] != nil)
+              DB.execute("UPDATE worker SET division_name='#{worker['division_name'].to_s}' WHERE name='#{worker['name'].to_s}'")
+            end
+  
+            if (worker['type']==nil || worker['type'].to_i==MyProcess::TYPE_LOCAL)
+              DB.execute("UPDATE worker SET type=#{MyProcess::TYPE_LOCAL.to_s} WHERE name='#{worker['name'].to_s}'")
+            else
+              DB.execute("UPDATE worker SET type=#{MyProcess::TYPE_REMOTE.to_s} WHERE name='#{worker['name'].to_s}'")
+            end
+          end
+          
+          # release resources
+          DB.disconnect
+          GC.start
+        }
+        
+      end
+    end
+  
+    # actualiza el rol y objeto asignado a este worker 
+    def updateFromCentral()
+      uri = URI("#{WORKER_API_SERVER_URL}/api1.3/pampa/get.json")
+      res = Net::HTTP.post_form(uri, {'api_key' => BlackStack::Pampa::api_key, 'name' => self.name})
+      parsed = JSON.parse(res.body)
+      if (parsed['status'] != "success")
+        raise parsed['status'].to_s
+      else  
+        # map response
+        self.id_division        = parsed['id_division']
+        self.assigned_process   = parsed['assigned_process']
+        self.id_object          = parsed['id_object']
+        self.division_name      = parsed['division_name']
+        self.active             = true
+  
+        if (parsed['id_object'].to_s.size>0)
+          aux_id_object = "'#{parsed['id_object']}'"
+        else
+          aux_id_object = "NULL"
+        end
+  
+        # NOTA: DEBO HACER EL UPDATE POR FUERA DE SQUEL, DEBIDO AL BUG DE MAPEO DE SEQUEL
+        q =
+        "UPDATE worker SET " +
+        "active=1, id_division='#{parsed['id_division']}', assigned_process='#{parsed['assigned_process'].to_s.gsub("'","''")}', id_object=#{aux_id_object}, division_name='#{parsed['division_name'].to_s.gsub("'","''")}' " +
+        "WHERE id='#{self.id}'"
+        DB.execute(q)      
+      end
+    end
+  
+    # actualiza el rol y objeto asignado a cada worker asignado a esta division 
+    def self.updateActivesFromCentral()
+      BlackStack::Worker.getActives().each { |worker|
+        worker.updateFromCentral()
+      }
+    end
+=end
+  
+    # 
+    def factory(name, params)
+      w = BlackStack::Worker.where(:name=>name).first
+      if (w==nil)
+        w = BlackStack::Worker.new
+      end
+      w.id                  = parsed['id']
+      w.name                = name
+      w.assigned_process    = parsed['assigned_process']
+      w.id_object           = parsed['id_object']
+      w.id_division         = parsed['id_division']
+      w.division_name       = parsed['division_name']
+      w.ws_url              = parsed['ws_url']
+      w.ws_port             = parsed['ws_port']
+      w.save
+    end
+  
+    # Retorna true si este worker esta corriendo en nuestros propios servidores, 
+    # Retorna false si este worker esta correiendo en otro host, asumiendo que es el host del cliente.
+    # Comparando la pulic_ip_address del worer con la lista en BlackStack::Pampa::set_farm_external_ip_addresses.  
+    def hosted?
+      BlackStack::Pampa::set_farm_external_ip_addresses.inlude?(self.public_ip_address)
+    end # hosted?
+    
+    # Si es un worker hosteado en nuestos servidores (ver metodo hosted?), 
+    # => retorna la cantidad de dias que fa
+    def expirationDesc
+      s = "(unknown)"
+      if self.hosted?
+        if !self.expiration_time.nil?
+          s = DB["SELECT DATEDIFF(mi, GETDATE(), w.expiration_time) AS n FROM worker w WHERE w.id='#{self.id}'"].first[:n].to_i.to_time_spent
+        end 
+      else # no hosted
+        s = "(self-hosted)"
+      end
+      s
+    end 
+  
+    # Retorna la cantidad de minutos desde que este worker envio una senial de vida. 
+    # Este metodo se usa para saber si un worker esta activo o no.
+    def last_ping_minutes()
+      q = "SELECT DATEDIFF(mi, p.last_ping_time, getdate()) AS minutes FROM worker p WHERE p.id='#{self.id}'"
+      return DB[q].first[:minutes].to_i
+    end
+
+    # returns true if this worker had got a ping within the last 5 minutes 
+    def active?
+      self.last_ping_minutes < BlackStack::BaseWorker::KEEP_ACTIVE_MINUTES
+    end
+
+    # escribe en el archivo de log de este worker
+    def log(s, level=1, is_error=false)
+      logw(s, self.process, self.id, level, is_error)
+    end
+  
+    # envia una senial de vida a la division
+    # TODO: guardar fecha-hora del ultimo ping en un atributo privado, y evitar el acceso escesivo a la base de datos
+    def ping()
+      DB.execute("UPDATE worker SET last_ping_time=GETDATE() WHERE id='#{self.id}'")
+    end
+    
+    # DEPRECATED
+    def self.getActivesCount(processName)
+      raise "Method needs some code inside."
+    end
+  
+    # obtiene array de workers actives, filtrados por proceso y por tipo de worker.
+    def self.getActives(assigned_process_name=nil, worker_name_filter=nil)
+      a = Array.new
+      q = ""
+      if (assigned_process_name!=nil)
+        q =
+            "SELECT p.id AS [id] " +
+            "FROM worker p WITH (NOLOCK INDEX(IX_peer__process__last_ping_time)) " + 
+            "WHERE last_ping_time>DATEADD(mi,-5,GETDATE()) " +
+            "AND ISNULL(active,0)=1 " + # active indica si este worker fue asignado a esta division en la central
+            "AND assigned_process='#{assigned_process_name}' "
+  
+        if worker_name_filter != nil
+          q = q +
+            "AND p.name LIKE '%#{worker_name_filter.to_s}%' "
+        end
+        
+        q = q +
+            "ORDER BY p.name "
+        DB[q].all do |row|
+          a << BlackStack::Worker.where(:id=>row[:id]).first
+        end
+      else
+        q =
+            "SELECT p.id AS [id] " +
+            "FROM worker p WITH (NOLOCK INDEX(IX_peer__process__last_ping_time)) " + 
+            "WHERE last_ping_time>DATEADD(mi,-5,GETDATE()) " +
+            "AND ISNULL(active,0)=1 "
+  
+        if worker_name_filter != nil
+          q = q +
+            "AND p.name LIKE '%#{worker_name_filter.to_s}%' "
+        end
+  
+        q = q +
+            "ORDER BY p.name "
+        DB[q].all do |row|
+          a << BlackStack::Worker.where(:id=>row[:id]).first
+        end
+      end
+      
+      return a
+    end
+  
+    # obtiene cantidad de registros en cola para incrawl.lnsearchvariation
+    def getPendingLnSearchVariationBlockInCrawlCount()
+      return DB.from(:lnsearchvariationblock).where(:incrawl_reservation_id=>self.id, :incrawl_start_time=>nil).count
+    end
+  
+  end # class Worker
+  
+end # module BlackStack

metadata

@@ -0,0 +1,229 @@
+--- !ruby/object:Gem::Specification
+name: pampa_workers
+version: !ruby/object:Gem::Version
+  version: 0.0.38
+platform: ruby
+authors:
+- Leandro Daniel Sardi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-12-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: websocket
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.8
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+- !ruby/object:Gem::Dependency
+  name: tiny_tds
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.28.0
+- !ruby/object:Gem::Dependency
+  name: blackstack_commons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.20
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.20
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.20
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.20
+- !ruby/object:Gem::Dependency
+  name: simple_cloud_logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.16
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.16
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.16
+- !ruby/object:Gem::Dependency
+  name: simple_command_line_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+- !ruby/object:Gem::Dependency
+  name: simple_host_monitoring
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+description: 'THIS GEM IS STILL IN DEVELOPMENT STAGE. Find documentation here: https://github.com/leandrosardi/pampa.'
+email: leandro.sardi@expandedventure.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/basedivision.rb
+- lib/baseworker.rb
+- lib/client.rb
+- lib/division.rb
+- lib/login.rb
+- lib/mybotprocess.rb
+- lib/mychildprocess.rb
+- lib/mycrawlprocess.rb
+- lib/mylocalprocess.rb
+- lib/myparentprocess.rb
+- lib/myprocess.rb
+- lib/myremoteprocess.rb
+- lib/pampa-local.rb
+- lib/pampa.rb
+- lib/params.rb
+- lib/remotedivision.rb
+- lib/remoteworker.rb
+- lib/role.rb
+- lib/timezone.rb
+- lib/user.rb
+- lib/userdivision.rb
+- lib/userrole.rb
+- lib/worker.rb
+homepage: https://rubygems.org/gems/pampa
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: THIS GEM IS STILL IN DEVELOPMENT STAGE. Ruby library for distributing computing,
+  supporting dynamic reconfiguration, distribution of the computation jobs, error
+  handling, job-retry and fault tolerance, fast (non-direct) communication to ensure
+  real-time capabilities.
+test_files: []