wakame 0.4.2 → 0.5.0

data/lib/wakame/status_db.rb

@@ -0,0 +1,383 @@
+
+require 'thread'
+
+module Wakame
+  module StatusDB
+
+    def self.pass(&blk)
+      if Thread.current == WorkerThread.worker_thread
+        blk.call
+      else
+        WorkerThread.queue.enq(blk)
+      end
+    end
+    
+    def self.barrier(&blk)
+      abort "Cant use barrier() in side of the EventMachine thread." if Kernel.const_defined?(:EventMachine) && ::EventMachine.reactor_thread?
+
+      if Thread.current == WorkerThread.worker_thread
+        return blk.call
+      end
+      
+      @q ||= ::Queue.new
+      time_start = ::Time.now
+      
+      self.pass {
+        begin
+          res = blk.call
+          @q << [true, res]
+        rescue => e
+          @q << [false, e]
+        end
+      }
+      
+      res = @q.shift
+      time_elapsed = ::Time.now - time_start
+      Wakame.log.debug("#{self}.barrier: Elapsed time for #{blk}: #{time_elapsed} sec") if time_elapsed > 0.05
+      if res[0] == false && res[1].is_a?(Exception)
+        raise res[1]
+      end
+      res[1]
+    end
+    
+    class WorkerThread
+
+      def self.queue
+        @queue ||= ::Queue.new
+      end
+
+      def self.worker_thread
+        @thread 
+      end
+
+      def self.init
+        @proceed_reqs = 0
+
+        if @thread.nil?
+          @thread = Thread.new {
+            while blk = queue.deq
+              begin
+                blk.call
+              rescue => e
+                Wakame.log.error("#{self.class}: #{e}")
+                Wakame.log.error(e)
+              end
+              @proceed_reqs += 1
+            end
+          }
+        end
+      end
+
+      def self.terminate
+        if self.queue.size > 0
+          Wakame.log.warn("#{self.class}: #{self.class.queue.size} of non-processed reqs are going to be ignored to shutdown the worker thread.")
+          self.queue.clear
+        end
+        self.worker_thread.kill if !self.worker_thread.nil? && self.worker_thread.alive?
+      end
+    end
+
+
+    def self.adapter
+      @adapter ||= SequelAdapter.new
+    end
+
+    class SequelAdapter
+      DATA_FORMAT_VERSION='0.4'
+
+      def initialize
+        require 'sequel/core'
+        require 'sequel/model'
+
+        #@db = Sequel.connect(Wakame.config.status_db_dsn, {:logger=>Wakame.log})
+        @db = Sequel.connect(Wakame.config.status_db_dsn)
+        
+        if [:metadata, :model_stores].all?{ |i| @db.table_exists?(i) }
+          m = @db[:metadata].where(:id=>1).first
+
+          unless m && m[:version] == DATA_FORMAT_VERSION
+            setup_store
+          end
+
+        else
+          setup_store
+        end
+
+        # Generate Sequel::Model class dynamically.
+        # This is same as below:
+        # class ModelStore < Sequel::Model
+        #    unrestrict_primary_key
+        # end
+        @model_class = Class.new(Sequel::Model(:model_stores)) { |klass|
+          klass.unrestrict_primary_key
+        }
+        @model_class.plugin :schema
+        @model_class.plugin :hook_class_methods
+        @model_class.class_eval {
+          before_create(:set_created_at) do
+            self.updated_at = self.created_at = Time.now
+          end
+          before_update(:set_updated_at) do
+            self.updated_at = Time.now
+          end
+        }
+        # @model_class.plugin :caching, store
+      end
+
+      def setup_store
+        @db.drop_table :metadata rescue nil
+        @db.create_table? :metadata do
+          primary_key :id
+          column :version, :string
+          column :created_at, :datetime
+        end
+
+        @db[:metadata].insert(:version=>DATA_FORMAT_VERSION, :created_at=>Time.now)
+
+        @db.drop_table :model_stores rescue nil
+        @db.create_table? :model_stores do
+          primary_key :id, :string, :size=>50, :auto_increment=>false
+          column :class_type, :string
+          column :dump, :text
+          column :created_at, :datetime
+          column :updated_at, :datetime
+        end
+      end
+
+      def find(id, &blk)
+        m = @model_class[id]
+        if m
+          hash = eval(m[:dump])
+          blk.call(id, hash)
+        end
+      end
+
+      # Find all rows belong to given klass name.
+      # Returns id list which matches class_type == klass
+      def find_all(klass)
+        ds = @model_class.where(:class_type=>klass.to_s)
+        ds.all.map {|r| r[:id] }
+      end
+
+      def exists?(id)
+        !@model_class[id].nil?
+      end
+
+      def save(id, hash)
+        m = @model_class[id]
+        if m.nil? 
+          m = @model_class.new
+          m.id = id
+          m.class_type = hash[AttributeHelper::CLASS_TYPE_KEY]
+       end 
+        m.dump = hash.inspect
+        m.save
+      end
+
+      def delete(id)
+        @model_class[id].destroy
+      end
+
+      def clear_store
+        setup_store
+      end
+      
+    end
+
+
+    class Model
+     include ::AttributeHelper
+
+      module ClassMethods
+        def enable_cache
+          unless @enable_cache
+            @enable_cache = true
+            @_instance_cache = {}
+          end
+        end
+
+        def disable_cache
+          if @enable_cache
+            @enable_cache = false
+            @_instance_cache = {}
+          end
+        end
+
+        def _instance_cache
+          return {} unless @enable_cache
+
+          @_instance_cache ||= {}
+        end
+
+        def find(id)
+          raise "Can not retrieve the data with nil." if id.nil?
+          obj = _instance_cache[id]
+          return obj unless obj.nil?
+
+          StatusDB.adapter.find(id) { |id, hash|
+            if hash[AttributeHelper::CLASS_TYPE_KEY]
+              klass_const = Util.build_const(hash[AttributeHelper::CLASS_TYPE_KEY])
+            else
+              klass_const = self.class
+            end
+
+            # klass_const class is equal to self class or child of self class
+            if klass_const <= self
+              obj = klass_const.new
+            else
+              raise "Can not instanciate the object #{klass_const.to_s} from #{self}"
+            end
+
+            obj.on_before_load
+
+            obj.instance_variable_set(:@id, id)
+            obj.instance_variable_set(:@_orig, hash.dup.freeze)
+            obj.instance_variable_set(:@load_at, Time.now)
+        
+            hash.each { |k,v|
+              obj.instance_variable_set("@#{k}", v)
+            }
+
+            obj.on_after_load
+          }
+
+          _instance_cache[id] = obj
+          obj
+        end
+
+
+        def find_all
+          StatusDB.adapter.find_all(self.to_s).map { |id|
+            find(id)
+          }
+        end
+
+        def exists?(id) 
+          _instance_cache.has_key?(id) || StatusDB.adapter.exists?(id)
+        end
+
+        # A helper method to define an accessor with persistent flag.
+        def property(key, opts={})
+          case opts 
+          when Hash
+            opts.merge!({:persistent=>true})
+          else
+            opts = {:persistent=>true}
+          end
+          def_attribute(key.to_sym, opts)
+        end
+
+        def delete(id)
+          obj = find(id)
+          if obj
+            obj.on_before_delete
+
+            StatusDB.adapter.delete(id)
+            _instance_cache.delete(id)
+
+            obj.on_after_delete 
+          end
+        end
+
+      end
+
+      def self.inherited(klass)
+        klass.extend(ClassMethods)
+        klass.class_eval {
+          #include(::AttributeHelper)
+          #enable_cache
+
+          # Manually set attr option to get :id appeared in dump_attrs.
+          attr_attributes[:id]={:persistent=>false}
+        }
+      end
+
+      def id
+        @id ||= Wakame::Util.gen_id
+      end
+
+      def new_record?
+        @load_at.nil?
+      end
+
+      def dirty?(key=nil)
+        return true if new_record?
+
+        if key
+          attr_attr = self.class.get_attr_attribute(key.to_sym)
+          raise "#{key} is not the key to be saved" if attr_attr.nil? || !attr_attr[:persistent]
+          return @_orig[key.to_sym] != self.__send__(key.to_sym)
+        else
+          self.class.merged_attr_attributes.each { |k,v|
+            next unless v[:persistent]
+            #p "@_orig[#{k.to_sym}]=#{@_orig[k.to_sym].inspect}"
+            #p "@self.__send__(#{k.to_sym})=#{self.__send__(k.to_sym).inspect}"
+            return true if @_orig[k.to_sym] != self.__send__(k.to_sym)
+          }
+          return false
+        end
+      end
+
+      def save
+#        return unless dirty?
+#       raise "No change" unless dirty?
+
+        validate_on_save
+
+        self.class.merged_attr_attributes.each { |k,v|
+          next unless v[:persistent]
+          if dirty?(k) && v[:call_after_changed]
+            case v[:call_after_changed]
+            when Symbol
+              self.__send__(v[:call_after_changed].to_sym) # if self.respond_to?(v[:call_after_changed].to_sym)
+            when Proc
+              v[:call_after_changed].call(self)
+            end
+          end
+        }
+
+        hash_saved = self.dump_attrs { |k,v,dumper|
+          if v[:persistent] == true
+            dumper.call(k)
+          end
+        }
+        @_orig = hash_saved.dup.freeze
+
+        StatusDB.adapter.save(self.id, hash_saved)
+      end
+
+      def delete
+        self.class.delete(self.id)
+      end
+
+      def reload
+        self.class._instance_cache.delete(self.id)
+        self.class.find(self.id)
+      end
+
+      # Callback methods
+      
+      # Called prior to copying data from database in self.find().
+      def on_before_load
+      end
+      # Called after copying data from database in self.find().
+      def on_after_load
+      end
+
+      def on_before_delete
+      end
+
+      def on_after_delete
+      end
+
+      protected
+
+      def validate_on_save
+      end
+
+
+    end
+    
+  end
+  
+end

data/lib/wakame/template.rb

@@ -1,6 +1,7 @@
 
 require 'fileutils'
 require 'erb'
+require 'pathname'
 
 module Wakame
   class Template
@@ -17,40 +18,44 @@ module Wakame
       @service_instance.resource.basedir
     end
 
+    def glob_basedir(glob_patterns, &blk)
+      glob_patterns = [glob_patterns] if glob_patterns.is_a? String
+
+      basedir_obj = Pathname.new(basedir)
+      paths = glob_patterns.collect {|pattern| Pathname.glob(File.join(basedir, pattern)).collect {|path| path.relative_path_from(basedir_obj) } }
+      paths = paths.flatten.uniq
+
+      paths.each &blk if blk
+
+      paths
+    end
+
     def render_config
-      service_instance.property.render_config(self)
+      service_instance.resource.render_config(self)
     end
 
     def cleanup
       FileUtils.rm_r( @tmp_basedir, :force=>true)
     end
 
-    def render(args)
-      args = [args] if args.is_a? String
-      
-      args.each { |path|
-        update(path) { |buf|
-          ERB.new(buf, nil, '-').result(service_instance.export_binding)
-        }
+    def render(path)
+      update(path) { |buf|
+        ERB.new(buf, nil, '-').result(service_instance.export_binding)
       }
     end
 
-    def cp(args)
-      args = [args] if args.is_a? String
-      args.each { |fname|
-
-        destpath = File.expand_path(fname, @tmp_basedir)
-        FileUtils.mkpath(File.dirname(destpath)) unless File.directory?(File.dirname(destpath))
-
-        FileUtils.cp_r(File.join(basedir, fname),
-                       destpath,
-                       {:preserve=>true}
-                       )
-      }
+    def cp(path)
+      destpath = File.expand_path(path, @tmp_basedir)
+      FileUtils.mkpath(File.dirname(destpath)) unless File.directory?(File.dirname(destpath))
+      
+      FileUtils.cp_r(File.join(basedir, path),
+                     destpath,
+                     {:preserve=>true}
+                     )
     end
 
-    def chmod(fname, mode)
-      File.chmod(mode, File.join(@tmp_basedir, fname))
+    def chmod(path, mode)
+      File.chmod(mode, File.join(@tmp_basedir, path))
     end
 
     def load(path)
@@ -77,113 +82,5 @@ module Wakame
     end
     
   end
-
-  module Template2
-    class Base
-      attr_reader :tmp_output_basedir
-      
-      def initialize()
-      end
-
-
-      def pre_render
-        @tmp_output_basedir = File.expand_path(Util.gen_id, Wakame.config.config_tmp_root)
-        FileUtils.mkdir_p @tmp_output_basedir
-      end
-
-      def render(service_instance)
-      end
-
-      def post_render
-        
-      end
-
-      # Returns source path which to be synced to the agent host
-      def sync_src
-      end
-
-
-      def load(path)
-        path = path.sub(/^\//, '')
-
-        return File.readlines(File.expand_path(path, Wakame.config.config_template_root), "r").join('')
-      end
-      
-      def save(path, buf)
-        path = path.sub(/^\//, '')
-
-        destpath = File.expand_path(path, @tmp_output_basedir)
-        FileUtils.mkpath(File.dirname(destpath)) unless File.directory?(File.dirname(destpath))
-
-        File.open(destpath, "w", 0644) {|f|
-          f.write(buf)
-        }
-      end
-      
-      def update(path, &blk)
-        buf = load(path)
-        buf = yield buf if block_given?
-        save(path, buf)
-      end
-
-
-    end
-
-    class MySQLTemplate  < Base
-      def render(service_instance)
-        require 'erb'
-        
-        FileUtils.mkpath File.expand_path('mysql', tmp_output_basedir)
-
-        ["mysql/my.cnf"].each { |path|
-          update(path) { |buf|
-            ERB.new(buf, nil, '-').result service_instance.export_binding
-          }
-        }
-      end
-
-      
-      def sync_src
-        File.join(@tmp_output_basedir, 'mysql')
-      end
-    end
-
-    class ApacheTemplate  < Base
-      attr_accessor :suffix
-
-      def initialize(suffix)
-        super()
-        @suffix = suffix
-      end
-      
-      def render(service_instance)
-#p service_instance.service_cluster.virtual_hosts
-        require 'erb'
-        
-        FileUtils.mkpath File.expand_path('apache2', tmp_output_basedir)
-
-        ["envvars-#{@suffix}", "apache2.conf"].each { |fname|
-          FileUtils.cp_r(File.join(Wakame.config.config_template_root, "apache2/#{fname}"),
-                         File.join(tmp_output_basedir, "apache2/#{fname}"),
-                         {:preserve=>true})
-        }
-
-        ["apache2/system-#{@suffix}.conf", "apache2/sites-#{@suffix}.conf"].each { |path|
-          update(path) { |buf|
-            ERB.new(buf, nil, '-').result service_instance.export_binding
-          }
-        }
-      end
-
-      
-      def sync_src
-        File.join(@tmp_output_basedir, 'apache2')
-      end
-      
-    end
-
-
-  end
-
   
 end

data/lib/wakame/trigger.rb

@@ -5,36 +5,28 @@ module Wakame
 
     def_attribute :enabled, true
 
-    attr_reader :rule_engine
-    def command_queue
-      @rule_engine.command_queue
-    end
-
-    def service_cluster
-      @rule_engine.service_cluster
-    end
+    #def service_cluster
+    #  action_manager.service_cluster
+    #end
+    #alias :cluster :service_cluster
 
     def master
-      @rule_engine.master
+      Master.instance
     end
 
     def agent_monitor
-      @rule_engine.agent_monitor
+      master.agent_monitor
     end
 
-    def bind_engine(rule_engine)
-      @rule_engine = rule_engine
+    def command_queue
+      master.command_queue
     end
 
     def trigger_action(action)
-      rule_engine.create_job_context(self, action)
-      action.bind_trigger(self)
-      
-      rule_engine.run_action(action)
-      action.job_id
+      job_id = master.action_manager.trigger_action(action)
     end
 
-    def register_hooks
+    def register_hooks(service_cluster_id)
     end
 
     def cleanup

data/lib/wakame/triggers/instance_count_update.rb

@@ -5,7 +5,7 @@ module Wakame
         rule.service_cluster.status == Service::ServiceCluster::STATUS_ONLINE
       }
 
-      def register_hooks
+      def register_hooks(cluster_id)
         @instance_counters = {}
         ms = Scheduler::PerHourSequence.new
         #ms[0]=1

data/lib/wakame/triggers/load_history.rb

@@ -42,7 +42,7 @@ module Wakame
         @history_period = 3
       end
       
-      def register_hooks
+      def register_hooks(cluster_id)
         event_subscribe(Event::AgentMonitored) { |event|
           @agent_data[event.agent.agent_id]={:load_history=>[], :last_event=>:normal}
           service_cluster.properties.each { |klass, prop|

data/lib/wakame/triggers/maintain_ssh_known_hosts.rb

@@ -17,11 +17,14 @@ module Wakame
 
           tmpfile = File.expand_path(File.basename(Wakame.config.ssh_known_hosts) + '.tmp', basedir)
           File.open(tmpfile, 'w') { |f|
-            agent_monitor.registered_agents.each { |k, agent|
-              host_keys.each { |k|
-                f << "#{Wakame::Util.ssh_known_hosts_hash(agent.agent_ip)} #{k}\n"
+            StatusDB.barrier do
+              agent_monitor.agent_pool.group_active.keys.each { |agent_id|
+                agent = Service::Agent.find(agent_id)
+                host_keys.each { |k|
+                  f << "#{Wakame::Util.ssh_known_hosts_hash(agent.agent_ip)} #{k}\n"
+                }
               }
-            }
+            end
           }
 
           FileUtils.move(tmpfile, Wakame.config.ssh_known_hosts, {:force=>true})
@@ -29,7 +32,7 @@ module Wakame
       end
 
 
-      def register_hooks
+      def register_hooks(cluster_id)
         event_subscribe(Event::AgentMonitored) { |event|
           trigger_action(UpdateKnownHosts.new)
         }

data/lib/wakame/triggers/shutdown_unused_vm.rb

@@ -1,7 +1,7 @@
 module Wakame
   module Triggers
     class ShutdownUnusedVM < Trigger
-      def register_hooks
+      def register_hooks(cluster_id)
         event_subscribe(Event::AgentPong) { |event|
           if event.agent.services.empty? &&
               Time.now - event.agent.last_service_assigned_at > Wakame.config.unused_vm_live_period &&