bsl-thor 0.0.2

data/lib/ThorMaster.rb

@@ -0,0 +1,226 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+# Options parsing gems
+require 'optparse'
+require 'pp'
+
+# YAML
+require 'yaml'
+
+# Rubygems
+require 'rubygems'
+
+# Amqp Gems
+require 'amqp'
+#require 'mq'
+
+# Event machine
+require 'eventmachine'
+
+# DB Related stuff
+require 'active_record'
+require 'pg'
+
+# Custom gems
+require File.join(File.dirname(__FILE__), 'ThorApplication.rb')
+require 'Bsl'
+
+module Thor
+  class AppMaster < Thor::Application
+    attr_accessor :options, :sql_connection, :request_exit
+
+    def initialize(opts = {})		
+      super(opts)
+
+      @@AMQP_DEFAULT_RETRY_INTERVAL = 3
+      @@AMQP_MAX_RETRY_INTERVAL = (30)
+      @@AMQP_MAX_RETRY_ATTEMPS = -1
+      @@AMQP_RETRY_MULTIPLER = 1.5
+
+      @amqp_retry_interval = @@AMQP_DEFAULT_RETRY_INTERVAL
+      @amqp_retry_attempt = 0
+
+      # Signalizes that application wants exit for some reason		
+      @request_exit = false
+      
+      # AMQP options
+      @options[:amqp_host] = "localhost"
+      @options[:amqp_port] = 1234
+      @options[:amqp_user] = "user"
+      @options[:amqp_password] = "password"
+      @options[:amqp_vhost] = "my-vhost"
+
+      @options[:sql_adapter] = "postgresql"
+      @options[:sql_host] = "host"
+      @options[:sql_port] = 5432
+      @options[:sql_user] = "user"
+      @options[:sql_password] = "password"
+      @options[:sql_database] = "database"
+      
+      @sql_connection = nil
+      
+      initialize_optparser { |opts|
+        ############################
+        # AMQP Section
+        ############################
+        # AMQP Host
+        opts.on( '-H', '--amqp-host STRING', "AMQP Server hostname") do |host|
+          @options[:amqp_host] = host
+        end
+
+        # AMQP Port
+        opts.on( '-p', '--amqp-port NUM', "AMQP Server port number") do |port|
+          @options[:amqp_port] = port
+        end
+
+        # AMQP Username
+        opts.on( '-u', '--amqp-user STRING', "AMQP Username") do |user|
+          @options[:amqp_user] = user
+        end
+
+        # AMQP Password
+        opts.on( '-P', '--amqp-password STRING', "AMQP Password") do |password|
+          @options[:amqp_password] = password
+        end
+
+        # AMQP Vhost
+        opts.on( '-V', '--amqp-vhost STRING', "AMQP Virtual Host") do |vhost|
+          @options[:amqp_vhost] = vhost
+        end
+        
+        ############################
+        # Sql section
+        ############################
+        # SQL Adapter
+        opts.on( '-sA', '--sql-adapter STRING', "SQL Adapter") do |adapter|
+          @options[:sql_adapter] = adapter
+        end
+        
+        # SQL Host
+        opts.on( '-sH', '--sql-host STRING', "SQL Server hostname") do |host|
+          @options[:sql_host] = host
+        end
+
+        # SQL Port
+        opts.on( '-sp', '--sql-port NUM', "SQL Server port number") do |port|
+          @options[:sql_port] = port
+        end
+
+        # SQL Username
+        opts.on( '-su', '--sql-user STRING', "SQL Username") do |user|
+          @options[:sql_user] = user
+        end
+
+        # SQL Password
+        opts.on( '-sP', '--sql-password STRING', "SQL Password") do |password|
+          @options[:sql_password] = password
+        end
+        
+        # SQL Database
+        opts.on( '-sD', '--sql-database STRING', "SQL Database") do |database|
+          @options[:sql_adapter] = database
+        end
+      }
+    end
+    
+      # Resets internal AMQP connection failure counter/interval
+    def amqp_reset_retry_interval
+      @amqp_retry_interval = @@AMQP_DEFAULT_RETRY_INTERVAL
+      @amqp_retry_attempt = 0
+    end
+
+    # Starts AMQP connection
+    def amqp_start
+      Bsl::Logger::Log "Starting AMQP - Connecting #{options[:amqp_user]}@#{options[:amqp_host]}:#{options[:amqp_port]}#{options[:amqp_vhost]}"		
+      AMQP.start(:host => options[:amqp_host], :port => options[:amqp_port],  :vhost => options[:amqp_vhost], :user => options[:amqp_user], :password => options[:amqp_password] ) do
+        amqp_reset_retry_interval()
+      end
+    end
+
+    # Stops Running AMQP connection
+    def amqp_stop
+      AMQP.stop { EM.stop }
+    end
+    
+    # Connects to SQL
+    def sql_start
+      Bsl::Logger::Log "Connecting to SQL - Connecting #{options[:sql_adapter]}://#{options[:sql_user]}@#{options[:sql_host]}:#{options[:sql_port]}"		
+      
+      
+      begin
+        ActiveRecord::Base.establish_connection(
+          {
+            :adapter  => options[:sql_adapter],
+            :host     => options[:sql_host],
+            :username => options[:sql_user],
+            :password => options[:sql_password],
+            :database => options[:sql_database]
+          }
+        )
+        @sql_connection = ActiveRecord::Base.connection
+      rescue Exception => e
+        Bsl::Logger::Log "Could not connect to DB, reason: #{e.message}!"
+        return false
+      end
+
+      return @sql_connection != nil
+    end
+    
+    # Stop SQL connection
+    def sql_stop
+      
+    end
+
+    # Handles failure when connecting to AMQP
+    def amqp_handle_failure(e)
+      amqp_stop()
+
+      max_attempts_reached = false
+      if(@@AMQP_MAX_RETRY_ATTEMPS != nil && @@AMQP_MAX_RETRY_ATTEMPS >= 0)
+        @amqp_retry_attempt = @amqp_retry_attempt + 1
+        max_attempts_reached = @amqp_retry_attempt > @@AMQP_MAX_RETRY_ATTEMPS
+      end
+
+      if(max_attempts_reached == false)				
+        Bsl::Logger::Log "Unable to connect to AMQP: #{e.to_s}"
+        Bsl::Logger::Log "Next attempt in #{@amqp_retry_interval} sec(s)."
+
+        sleep (@amqp_retry_interval)
+        @amqp_retry_interval = @amqp_retry_interval * @@AMQP_RETRY_MULTIPLER
+        @amqp_retry_interval = @@AMQP_MAX_RETRY_INTERVAL if @amqp_retry_interval  > @@AMQP_MAX_RETRY_INTERVAL
+      else
+        if(@@AMQP_MAX_RETRY_ATTEMPS != nil)
+          Bsl::Logger::Log "Maximum AQMP reconnect attempts limit reached (#{@@AMQP_MAX_RETRY_ATTEMPS}), quitting."
+        end
+        @request_exit = true
+      end
+    end
+
+    # Main entry-point
+    def main
+      super()
+      
+      if(sql_start() == false)
+        Bsl::Logger::Log "Unable to connect to DB, quitting!"
+        exit
+      end
+      
+      # Run loop while exit is not requested
+      while(@request_exit == false)
+        begin
+          amqp_start()
+        rescue Exception => e
+          amqp_handle_failure(e)
+        end
+      end
+      
+      sql_stop()
+    end
+  end
+end
+
+if $0 == __FILE__
+	Thor::AppMaster.new.main
+end

data/lib/ThorNode.rb

@@ -0,0 +1,203 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+# Options parsing gems
+require 'optparse'
+require 'pp'
+
+# YAML
+require 'yaml'
+
+# Rubygems
+require 'rubygems'
+
+# Amqp Gems
+require 'amqp'
+#require 'mq'
+
+# Event machine
+require 'eventmachine'
+
+require 'socket'
+
+require 'digest/md5'
+
+# Custom gems
+require File.join(File.dirname(__FILE__), 'ThorApplication.rb')
+require File.join(File.dirname(__FILE__), 'ThorUtils.rb')
+require 'Bsl'
+
+module Thor
+	class Node < Thor::Application
+		attr_accessor :request_exit, :jobs, :jobs_lock
+
+		StructJob = Struct.new(:klass, :path)
+
+		# C-tor    
+		def initialize(opts = {})
+			super(opts)
+			
+			@jobs = {}
+			@jobs_lock = Mutex.new
+
+			options[:jobs_dir] = File.join(File.dirname(__FILE__), 'jobs')
+			options[:boot_file] = ""
+
+			initialize_optparser { |opts|
+			############################
+			# AMQP Section
+			############################
+				# Jobs Dir
+				opts.on( '-j', '--jobs-dir STRING', "AMQP Virtual Host") do |jobs_dir|
+					options[:jobs_dir] = jobs_dir
+				end
+
+				# Boot file
+				opts.on( '-b', '--boot-file STRING', "Boot file specifying which jobs run automaticaly") do |boot_file|
+					options[:boot_file] = boot_file
+				end
+			}
+		end
+
+		def amqp_loop(amqp)
+			super(amqp)
+			
+			# Start boot time jobs, move somewhere else!
+			boot_file = options[:boot_file]
+			if(boot_file != nil && boot_file != "")
+				boot_jobs = load_boot_file(boot_file)
+				if(boot_jobs != nil)
+					start_boot_time_jobs(boot_jobs, amqp)
+				end
+			end
+
+			#amqp_exchange_create_direct(guid) # Create local direct exchange
+			#amqp_exchange_connect_master(options[:amqp_channel_master], guid)
+		end
+		
+		# Enumerate existing jobs
+		def enumerate_jobs(dir, exclude = [])
+			res_jobs = {}
+		
+			if(File.directory?(dir) == false)
+				return nil
+			end	
+
+			Dir[dir + "/*"].each do |path|
+				next if (File.directory?(path) == false)
+				basename = File.basename(path)
+				dash_index = basename.index("-")
+				if(dash_index == nil || dash_index < 0 || (basename.length - dash_index - 1) < 5)
+					Bsl::Logger::Log "Invalid job dir name - #{path}"
+					next
+				end
+
+				job_name = basename.slice(0,dash_index)
+				job_name_version = basename.slice(dash_index+1, basename.length - dash_index)
+
+				job_main = File.join(path + "/main.rb")
+				if(File.exists?(job_main) == false)
+					Bsl::Logger::Log "Job entry point does not exists - #{job_main}"
+					next
+				end
+
+				require job_main
+
+				job = nil
+				begin
+					job = Thor::Jobs::const_get(job_name)
+				rescue
+					Bsl::Logger::Log "Unable to find class 'Thor::Jobs::#{job_name}' in file '#{job_main}'"
+					next
+				end
+
+				job_author = job.author
+				job_description = job.description
+				job_license = job.license
+				job_version = job.version
+
+				if(job_version != job_name_version)
+					Bsl::Logger::Log "Job '#{job_name}' inconsistent versions, filename: '#{job_name_version}', class: '#{job_version}, skipping.'"
+					next
+				end
+
+				Bsl::Logger::Log "Adding job, name: '#{job_name}', version: '#{job_version}', basename: '#{basename}'"
+				res_jobs[basename] = StructJob.new(job, job_main)
+			end
+
+			return res_jobs
+		end
+		
+		# Load boot file if specified
+		def load_boot_file(boot_file)
+			Bsl::Logger::Log "Loading boot file '#{boot_file}'."
+			if(File.exists?(boot_file) == false)
+				Bsl::Logger::Log "Boot file '#{boot_file}' is not valid file path!"
+				return nil
+			end
+			
+      require boot_file
+      basename = File.basename(boot_file, ".*")
+      return ThorBoot::const_get(basename)::boot_jobs
+		end
+
+		# Starts boot time jobs
+		def start_boot_time_jobs(boot_time_jobs, amqp)
+			boot_time_jobs.each do |boot_time_job|
+				@jobs_lock.synchronize {
+					job_name = boot_time_job[:name]
+					job = @jobs[job_name]
+					if(job != nil)
+						job_klass = job[:klass]
+						
+						job_opts = options						
+						job_opts = job_opts.merge(boot_time_job[:options])						
+						job_opts = job_opts.merge({:amqp => {:conn => amqp}})	    
+						job_opts.merge!({:amqp => {:conn => amqp}}) # TODO: Use Thor::AmqpNode defaults
+						
+						if(options[:verbose])
+							Bsl::Logger::Log "Starting boot job '#{job_klass.name}'."
+							#Bsl::Logger::Log "Job options '#{job_opts.inspect}'."
+						end
+						
+						job_instance = job_klass.new(job_opts)
+						job_instance.start(job_opts)
+						Bsl::Logger::Log "Boot job '#{job_klass.name}' started."
+					else
+						Bsl::Logger::Log "Invalid boot job specified, name: '#{job_name}'"
+					end
+				}
+			end
+		end
+		
+		# Main entry-point
+		def main
+			super()
+
+			# Refresh jobs
+			new_jobs = enumerate_jobs(options[:jobs_dir])
+			@jobs_lock. synchronize {
+				@jobs = new_jobs
+			}
+
+			# Run loop while exit is not requested
+			while(@request_exit == false)
+				begin
+					amqp_start()
+				rescue SystemExit, Interrupt
+					Bsl::Logger::Log "Received interrupt, quitting!"
+					@request_exit = true
+					amqp_stop()
+				rescue Exception => e
+					amqp_handle_failure(e)
+				end
+				puts "."
+			end
+		end
+	end
+end
+
+if $0 == __FILE__
+	Thor::Node.new.main
+end

data/lib/ThorUtils.rb

@@ -0,0 +1,40 @@
+require 'Bsl'
+
+require 'sys/admin'
+require 'sys/proctable'
+require 'sys/uname'
+require 'sys/cpu'
+require 'sys/uptime'
+require 'net/proto'
+require 'sys/host'
+require 'sys/filesystem'
+
+module Thor
+  
+  # Loads all ActiveRecord models
+  def self.require_all_models(logit = false)
+    dir = File.join(File.dirname(__FILE__), 'models/')
+    Dir[dir + '*.rb'].each do |file|
+      if(logit)
+        Bsl::Logger::Log "Including model '#{file}'"
+      end
+      require file
+    end
+  end
+  
+  # Generates unique exchange name
+  # Format is following IP.HOSTNAME.PID.TID(INST_HASH)
+  # PID = Process ID
+  # TID = Thread ID
+  # INST_HASH = Hash from instance pointer if some instance is specified as param
+  def self.generate_guid(inst = nil)
+    res = ""
+    
+    ip = Sys::Host.ip_addr.to_s.gsub(".", "-")
+    hostname = Sys::Host.hostname.gsub(".", "--")
+    pid = Process.pid
+    tid = Thread.current.object_id
+    instance = inst != nil ? ".#{instance.to_s}" :  ""
+    return "#{ip.to_s}.#{hostname}.#{pid}.#{tid}#{instance}"
+  end
+end

data/lib/ThorWorker.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+# Options parsing gems
+require 'optparse'
+require 'pp'
+
+# YAML
+require 'yaml'
+
+# Rubygems
+require 'rubygems'
+
+# Event machine
+require 'eventmachine'
+
+# Custom gems
+require File.join(File.dirname(__FILE__), 'ThorApplication.rb')
+require 'Bsl'
+
+module Thor
+  class AppWorker < Thor::Application
+    attr_accessor :options, :optpars
+
+    def initialize(opts = {})
+      super(opts)
+
+      # Event Machine options
+      options[:em_port] = 8467 # Thor on cell-phone keyboard
+      options[:em_auth_token] = ""
+
+      initialize_optparser { |opts|
+        ############################
+        # EventMachine Section
+        ############################
+        # EM Port
+        opts.on( '-ep', '--em-port NUM', "EventMachine port") do |port|
+          options[:em_port] = port
+        end
+
+        # EM Authentication token
+        opts.on( '-eat', '--em-auth-token STRING', "Authentication token used for communication with EM") do |token|
+          options[:em_auth_token] = token
+        end
+      }
+    end
+
+    # Main entry-point
+    def main
+      super()
+    end
+  end
+end
+
+if $0 == __FILE__
+	Thor::AppWorker.new.main
+end

data/lib/boot/Boot.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+module ThorBoot  
+  class Boot
+    # Default AMQP Options
+    @@DEFAULT_AMQP = {
+      :amqp => {
+       :host => "amqp_host",
+       :port => 5672,
+       :user => "amqp_user",
+       :password => "amqp_password",
+       :vhost => "amqp_vhost"
+      }    
+    }
+    
+    # Default EventMachine Options
+    @@DEFAULT_EM = {
+      
+    }
+    
+    # Default SQL Options
+    @@DEFAULT_SQL = {
+      :sql => {
+       :host => "sql_host",
+       :port => 5432,
+       :user => "sql_user",
+       :password => "sql_password",
+       :schema => "public"
+      } 
+    }
+    
+    # Return default AMQP Options
+    def self.default_options_amqp
+      return @@DEFAULT_AMQP
+    end
+    
+    # Returns default Event Machine Options
+    def self.default_options_em
+      return @@DEFAULT_EM
+    end
+    
+    # Returns default SQL Options
+    def self.default_options_sql
+      return @@DEFAULT_SQL
+    end
+    
+    # Return merged default Options
+    def self.default_options
+      res = {}
+      
+      res.merge!(default_options_amqp)
+      res.merge!(default_options_em)
+      res.merge!(default_options_sql)
+      
+      return res
+    end
+    
+    # Merges options with default options
+    def self.merge_options(opts)
+      return default_options.merge(opts)
+    end
+    
+    # Default function for getting boot jobs
+    def self.boot_jobs
+      boot_jobs = []
+      return boot_jobs
+    end
+  end
+end
+

data/lib/boot/Client.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+require File.join(File.dirname(__FILE__), 'Node.rb')
+
+module ThorBoot
+  class Client < Node
+    def self.boot_jobs
+      opts_node = {:channel_nodes => "clients"}
+      opts_client = {}
+      opts_monitor_client = {
+        :channel_request => "monitor.request",
+        :channel_response => "monitor.response"  
+      }
+      boot_jobs = [
+        {:name => "Node-0.0.1", :options => merge_options(opts_node)},
+        {:name => "Client-0.0.1", :options => merge_options(opts_client)},
+        {:name => "MonitorClient-0.0.1", :options => merge_options(opts_monitor_client)}
+      ]
+      return (super() + boot_jobs)
+    end
+  end
+end
+

data/lib/boot/Master.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+require File.join(File.dirname(__FILE__), 'Node.rb')
+
+module ThorBoot
+  class Master < Node
+    def self.boot_jobs
+      opts_node = {:channel_nodes => "masters"}
+      opts_master = {}
+      opts_monitor_master = {
+        :channel_request => "monitor.request",
+        :channel_response => "monitor.response",
+        :request_interval => 1
+      }
+      boot_jobs = [
+        {:name => "Node-0.0.1", :options => merge_options(opts_node)},
+        {:name => "Master-0.0.1", :options => merge_options(opts_master)},
+        {:name => "MonitorMaster-0.0.1", :options => merge_options(opts_monitor_master)}  
+      ]
+      return (super() + boot_jobs)
+    end
+  end
+end
+

data/lib/boot/Node.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+$KCODE="UTF8"
+
+require File.join(File.dirname(__FILE__), 'Boot.rb')
+
+module ThorBoot
+  class Node < Boot
+    def self.boot_jobs
+      opts = {:channel_nodes => "nodes"}
+      boot_jobs = [{:name => "Node-0.0.1", :options => merge_options(opts)}]
+      return (super() + boot_jobs)
+    end
+  end
+end
+

data/lib/config/ThorClient.yml.template

@@ -0,0 +1,12 @@
+verbose: false
+
+amqp_host: your-host.com
+amqp_port: 5672
+amqp_user: your_user
+amqp_password: your_password
+amqp_vhost: "your_vhost
+
+amqp_channel_master: "master"
+
+em_port: 8467
+em_auth_token: ""

data/lib/config/ThorCmd.yml.template

@@ -0,0 +1,4 @@
+verbose: false
+
+em_port: 8467
+em_auth_token: ""

data/lib/config/ThorMaster.yml.template

@@ -0,0 +1,19 @@
+verbose: false
+
+amqp_host: "host"
+amqp_port: 5672
+amqp_user: "user"
+amqp_password: "password"
+amqp_vhost: "vhost"
+
+sql_adapter: "postgresql"
+sql_host: "host"
+sql_port: 5432
+sql_user: "user"
+sql_password: "password"
+sql_database: "db"
+
+amqp_channel_master: "master"
+
+em_port: 8467
+em_auth_token: ""