wakame 0.5.0 → 0.5.1

data/lib/wakame/master_managers/cluster_manager.rb

@@ -0,0 +1,176 @@
+module Wakame
+  module MasterManagers
+   class ClusterManager
+     include MasterManager
+
+     class ClusterConfigLoader
+
+       def load(cluster_rb_path=Wakame.config.cluster_config_path)
+         Wakame.log.info("#{self.class}: Loading cluster.rb: #{cluster_rb_path}")
+         @loaded_cluster_names = {}
+
+         eval(File.readlines(cluster_rb_path).join(''), binding)
+
+         # Clear uninitialized cluster data in the store.
+         Service::ServiceCluster.find_all.each { |cluster|
+           cluster.delete unless @loaded_cluster_names.has_key?(cluster.name)
+         }
+
+         @loaded_cluster_names
+       end
+
+
+       private
+       def define_cluster(name, &blk)
+         cluster = Service::ServiceCluster.find(Service::ServiceCluster.id(name))
+         if cluster.nil?
+           cluster = Service::ServiceCluster.new
+           cluster.name = name
+         end
+
+         Models::AgentPool.reset
+         cluster.reset
+
+         blk.call(cluster)
+
+         cluster.save
+
+         Wakame.log.info("#{self.class}: Loaded Service Cluster: #{cluster.name}")
+         @loaded_cluster_names[name]=cluster.id
+       end
+
+     end
+
+     def init
+       # Periodical cluster status updater
+       @status_check_timer = EM::PeriodicTimer.new(5) {
+         StatusDB.pass {
+           clusters.each { |cluster_id|
+             Service::ServiceCluster.find(cluster_id).update_cluster_status
+           }
+         }
+       }
+       
+       # Event based cluster status updater
+       @check_event_tickets = []
+       [Event::ServiceOnline, Event::ServiceOffline, Event::ServiceFailed].each { |evclass|
+         @check_event_tickets << EventDispatcher.subscribe(evclass) { |event|
+           StatusDB.pass {
+             clusters.each { |cluster_id|
+               Service::ServiceCluster.find(cluster_id).update_cluster_status
+             }
+           }
+         }
+       }
+      
+       @check_event_tickets << EventDispatcher.subscribe(Event::ServiceStatusChanged) { |event|
+         svc = Service::ServiceInstance.find(event.instance_id)
+         case svc.status
+         when Service::STATUS_ENTERING
+           EM.defer {
+             # Refresh the monitoring conf on the agent
+             svc.cloud_host.live_monitors.each { |path, conf|
+               Wakame.log.debug("#{self.class}: Refreshing monitoring setting on #{svc.cloud_host.agent_id} (on enter): #{path} => #{conf.inspect}")
+               Master.instance.actor_request(svc.cloud_host.agent_id, '/monitor/reload', path, conf).request.wait
+             }
+           }
+         when Service::STATUS_QUITTING
+           EM.defer {
+             # Refresh the monitoring conf on the agent
+             svc.cloud_host.live_monitors.each { |path, conf|
+               Wakame.log.debug("#{self.class}: Refreshing monitoring setting on #{svc.cloud_host.agent_id} (on quit): #{path} => #{conf.inspect}")
+               Master.instance.actor_request(svc.cloud_host.agent_id, '/monitor/reload', path, conf).request.wait
+             }
+           }
+         end
+       }
+     end
+
+     def reload
+     end
+
+     def terminate
+       @status_check_timer.cancel
+       @check_event_tickets.each { |t|
+         EventDispatcher.unsubscribe(t)
+       }
+     end
+
+     def clusters
+       Models::ServiceClusterPool.all.map{|r| r.service_cluster_id }
+     end
+
+     def register(cluster)
+       raise ArgumentError unless cluster.is_a?(Service::ServiceCluster)
+       Models::ServiceClusterPool.register_cluster(cluster.name)
+     end
+
+     def unregister(cluster_id)
+       @clusters.delete(cluster_id)
+     end
+
+     def load_config_cluster
+       ClusterConfigLoader.new.load.each { |name, id|
+         Models::ServiceClusterPool.register_cluster(name)
+       }
+       resolve_template_vm_attr
+     end
+
+
+     private
+     def resolve_template_vm_attr
+       Models::ServiceClusterPool.each_cluster { |cluster|
+         cluster_id = cluster.id
+
+         if cluster.template_vm_attr.nil? || cluster.template_vm_attr.empty?
+           update_template_spec = lambda { |agent|
+             raise ArgumentError unless agent.is_a?(Service::Agent)
+
+             require 'right_aws'
+             ec2 = RightAws::Ec2.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key)
+             
+             ref_attr = ec2.describe_instances([agent.vm_attr[:aws_instance_id]])
+             ref_attr = ref_attr[0]
+                 
+             cluster = Service::ServiceCluster.find(cluster_id)
+             spec = cluster.template_vm_spec
+             Service::VmSpec::EC2.vm_attr_defs.each { |k, v|
+               spec.attrs[k] = ref_attr[v[:right_aws_key]]
+             }
+             cluster.save
+             
+             Wakame.log.debug("ServiceCluster \"#{cluster.name}\" template_vm_attr based on VM \"#{agent.vm_attr[:aws_instance_id]}\" : #{spec.attrs.inspect}")
+           }
+
+
+           agent_id = Models::AgentPool.instance.group_active.first
+           if agent_id.nil?
+             # Set a single shot event handler to set the template values up from the first connected agent.
+             EventDispatcher.subscribe_once(Event::AgentMonitored) { |event|
+               StatusDB.pass {
+                 update_template_spec.call(event.agent)
+               }
+             }
+           else
+             StatusDB.pass {
+               update_template_spec.call(Service::Agent.find(agent_id))
+             }
+           end
+         end
+
+         if cluster.advertised_amqp_servers.nil?
+           StatusDB.pass {
+             cluster = Service::ServiceCluster.find(cluster_id)
+             #cluster.advertised_amqp_servers = master.amqp_server_uri.to_s
+             attrs = Wakame::Master.ec2_fetch_local_attrs
+             cluster.advertised_amqp_servers = "amqp://#{attrs[:local_ipv4]}/"
+             cluster.save
+             Wakame.log.debug("ServiceCluster \"#{cluster.name}\" advertised_amqp_servers: #{cluster.advertised_amqp_servers}")
+           }
+         end
+
+       }
+     end
+   end
+ end
+end

data/lib/wakame/master_managers/command_queue.rb

@@ -0,0 +1,133 @@
+
+require 'uri'
+require 'thin'
+require 'thread'
+require 'json'
+
+module Wakame
+  module MasterManagers
+    class CommandQueue
+      include MasterManager
+
+      def initialize()
+        @queue = Queue.new
+        @result_queue = Queue.new
+        @statistics = {
+          :total_command_count => 0
+        }
+      end
+
+      def init
+        @command_thread = Thread.new {
+          Wakame.log.info("#{self.class}: Started command thread: #{Thread.current}")
+          while cmd = @queue.deq
+            begin
+              unless cmd.kind_of?(Wakame::Command)
+                Wakame.log.warn("#{self.class}: Incompatible type of object has been sent to ProcessCommand thread. #{cmd.class}")
+                next
+              end
+
+              res = nil
+              Wakame.log.debug("#{self.class}: Being processed the command: #{cmd.class}")
+              res = cmd.run
+              res
+            rescue => e
+              Wakame.log.error(e)
+              res = e
+            ensure
+              @result_queue.enq(res)
+            end
+          end
+        }
+
+        cmdsv_uri = URI.parse(Wakame.config.http_command_server_uri)
+
+        @thin_server = Thin::Server.new(cmdsv_uri.host, cmdsv_uri.port, Adapter.new(self))
+        @thin_server.threaded = true
+        @thin_server.start
+      end
+
+      def terminate
+        @thin_server.stop
+        @command_thread.kill
+      end
+
+      def send_cmd(cmd)
+        begin
+          @queue.enq(cmd)
+
+          ED.fire_event(Event::CommandReceived.new(cmd))
+
+          return @result_queue.deq()
+        rescue => e
+          Wakame.log.error("#{self.class}:")
+          Wakame.log.error(e)
+        end
+      end
+
+
+      class Adapter
+        
+        def initialize(command_queue)
+          @command_queue = command_queue
+        end
+        
+        def call(env)
+          req = Rack::Request.new(env)
+          begin
+            unless req.get?().to_s == "true"
+              raise "No Support Response"
+            end
+            query = req.query_string()
+            params = req.params()
+            if Wakame.config.enable_authentication == "true"
+              auth = authentication(params, query)
+            end
+            cname = params["action"].split("_")
+            begin
+              cmd = eval("Command::#{(cname.collect{|c| c.capitalize}).join}").new 
+              cmd.options = params
+              command = @command_queue.send_cmd(cmd)
+
+              if command.is_a?(Exception)
+                status = 500
+                body = json_encode(status, command.message)
+              else
+                status = 200
+                body = json_encode(status, "OK", command)
+              end
+            rescue => e
+              status = 404
+              body = json_encode(status, e)
+            end
+          rescue => e
+            status = 403
+            body = json_encode(status, e)
+            Wakame.log.error(e)
+          end
+          [ status, {'Content-Type' => 'text/javascript+json'}, body]
+        end
+
+        def authentication(path, query)
+          key = Wakame.config.private_key
+          req = query.split(/\&signature\=/)
+          hash = OpenSSL::HMAC::digest(OpenSSL::Digest::SHA256.new, key, req[0])
+          hmac = Base64.encode64(hash).gsub(/\+/, "").gsub(/\n/, "").to_s
+          if hmac != path["signature"]
+            raise "Authentication failed"
+          end
+        end
+
+        def json_encode(status, message, data=nil)
+          if status == 200 && data.is_a?(Hash)
+            body = [{:status=>status, :message=>message}, {:data=>data}].to_json
+          else
+            body = [{:status=>status, :message=>message}].to_json
+          end
+          body
+        end
+      end
+
+    end
+  end
+end

data/lib/wakame/models/agent_pool.rb

@@ -0,0 +1,113 @@
+
+module Wakame
+  module Models
+    class AgentPool < Sequel::Model
+      plugin :schema
+      plugin :hook_class_methods
+      
+      set_schema {
+        primary_key :id, :type => Integer
+        column :agent_pool_id, :int
+        column :agent_id, :varchar, :unique=>true
+        column :group_type, :int
+        column :created_at, :datetime
+        column :updated_at, :datetime
+      }
+
+      before_create(:set_created_at) do
+        self.updated_at = self.created_at = Time.now
+      end
+      before_update(:set_updated_at) do
+        self.updated_at = Time.now
+      end
+
+      GROUP_ACTIVE=1
+      GROUP_OBSERVED=2
+
+      DEFAULT_POOL_ID=1
+
+      def self.instance
+        self
+      end
+
+      def self.reset
+        filter(:agent_pool_id=>DEFAULT_POOL_ID).all.each { |row|
+          agent = Service::Agent.find(row[:agent_id])
+          if agent
+            agent.cloud_host_id = nil
+            agent.save
+          else
+            row.delete
+          end
+        }
+      end
+
+      def self.group_active
+        filter(:agent_pool_id=>DEFAULT_POOL_ID, :group_type=>GROUP_ACTIVE).all.map {|row| row[:agent_id] }
+      end
+
+      def self.group_observed
+        filter(:agent_pool_id=>DEFAULT_POOL_ID, :group_type=>GROUP_OBSERVED).all.map {|row| row[:agent_id] }
+      end
+
+      def self.agent_find_or_create(agent_id)
+        agent = Service::Agent.find(agent_id)
+        if agent.nil?
+          agent = Service::Agent.new
+          agent.id = agent_id
+          Wakame.log.debug("#{self.class}: Created new agent object with Agent ID: #{agent_id}")
+        end
+        agent
+      end
+
+      def self.register_as_observed(agent)
+        raise ArgumentError unless agent.is_a? Service::Agent
+        row = find_or_create(:agent_id=>agent.id, :agent_pool_id=>DEFAULT_POOL_ID)
+        if row[:group_type].nil? || row[:group_type] == GROUP_ACTIVE
+          # The agent is being registered at first time.
+          # Move the reference from active group to observed group.
+          row[:group_type]=GROUP_OBSERVED
+          row.save
+          Wakame.log.debug("#{self.class}: Register agent to observed group: #{agent.id}")
+        elsif 
+          row[:group_type]=GROUP_OBSERVED
+        end
+      end
+
+      def self.register(agent)
+        raise ArgumentError unless agent.is_a?(Service::Agent)
+        row = find_or_create(:agent_id=>agent.id, :agent_pool_id=>DEFAULT_POOL_ID)
+        if row[:group_type].nil? || row[:group_type] == GROUP_OBSERVED
+          # The agent is being registered at first time.
+          # Move the reference from observed group to the active group.
+          row[:group_type]=GROUP_ACTIVE
+          row.save
+          
+          Wakame.log.debug("#{self}: Register agent to active group: #{agent.id}")
+          ED.fire_event(Event::AgentMonitored.new(agent))
+        elsif row[:group_type] == GROUP_ACTIVE
+        end
+      end
+
+      def self.unregister(agent)
+        raise ArgumentError unless agent.is_a?(Service::Agent)
+        row = first(:agent_id=>agent.id, :agent_pool_id=>DEFAULT_POOL_ID)
+        if row.nil?
+        else
+          row.delete
+          Wakame.log.debug("#{self}: Unregister agent: #{agent.id}")
+          ED.fire_event(Event::AgentUnMonitored.new(agent))
+        end
+      end
+
+      def self.find_agent(agent_id)
+        raise "The agent ID \"#{agent_id}\" is not registered in the pool" unless first(:agent_id=>agent_id, :agent_pool_id=>DEFAULT_POOL_ID)
+        Service::Agent.find(agent_id) || raise("The agent ID #{agent_id} is registered. but not in the database.")
+      end
+
+    end
+  end
+  
+  Initializer.loaded_classes << Models::AgentPool if const_defined? :Initializer
+
+end

data/lib/wakame/models/application_repository.rb

@@ -0,0 +1,34 @@
+
+require 'sequel/model'
+
+module Wakame
+  module Models
+    class ApplicationRepository < Sequel::Model
+      plugin :schema
+      plugin :hook_class_methods
+      
+      set_schema {
+        primary_key :id, :type => Integer
+        column :app_name, :varchar, :unique=>true
+        column :repo_type, :varchar
+        column :repo_uri, :varchar
+        column :revision, :varchar
+        column :resource_tag, :varchar
+        column :options, :text
+        column :comment, :text
+        column :created_at, :datetime
+        column :updated_at, :datetime
+      }
+
+      before_create(:set_created_at) do
+        self.updated_at = self.created_at = Time.now
+      end
+      before_update(:set_updated_at) do
+        self.updated_at = Time.now
+      end
+
+    end
+  end
+
+  Initializer.loaded_classes << Models::ApplicationRepository if const_defined? :Initializer
+end