tengine_core 0.5.28

data/lib/tengine/core/find_by_name.rb

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+require 'tengine/core'
+
+module Tengine::Core::FindByName
+  extend ActiveSupport::Concern
+
+  class Error < Tengine::Errors::NotFound
+    attr_reader :klass, :name, :options
+    def initialize(klass, name, options = nil)
+      @klass, @name, @options = klass, name, options
+    end
+    def message
+      result = "#{klass.name} named #{name.inspect}"
+      result << " with #{options.inspect}" if options && !options.empty?
+      result << ' not found'
+      result
+    end
+  end
+
+  module ClassMethods
+    def find_by_name(name)
+      first(:conditions => {:name => name})
+    end
+
+    def find_by_name!(name, *args, &block)
+      result = find_by_name(name, *args, &block)
+      raise Error.new(self, name, args.last) unless result
+      result
+    end
+  end
+end

data/lib/tengine/core/handler.rb

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+require 'tengine/core'
+
+require 'tengine/event'
+
+# イベントハンドラ
+#
+# Tengineコアは、イベントを受信するとそのイベント種別名にマッチするイベントハンドラを探して
+# 見つかったイベントハンドラをすべて実行します。
+class Tengine::Core::Handler
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include Tengine::Core::CollectionAccessible
+  include Tengine::Core::SelectableAttr
+
+  # @attribute 実行するRubyのブロックが定義されているファイル名
+  field :filepath, :type => String
+
+  # @attribute 実行するRubyのブロックが定義されているファイルでの行番号
+  field :lineno  , :type => Integer
+
+  # @attribute 処理するイベントのイベント種別名の配列
+  field :event_type_names, :type => Array
+  array_text_accessor :event_type_names
+
+  # @attribute イベントが対象かどうかを判断するためのフィルタ定義
+  field :filter, :type => Hash, :default => {}
+  map_yaml_accessor :filter
+
+  # @attribute 実行対象の取得方法
+  field :target_instantiation_cd, :type => String, :default => '01'
+
+  selectable_attr :target_instantiation_cd do
+    entry '01', :binding        , "binding"
+    entry '02', :static         , "static"
+    entry '03', :instance_method, "instance_method"
+  end
+
+  # @attribute 実行対象となるメソッドの名前
+  field :target_method_name, :type => String
+
+
+  validates :filepath, :presence => true
+  validates :lineno  , :presence => true
+
+  embedded_in :driver, :class_name => "Tengine::Core::Driver"
+
+  def update_handler_path
+    event_type_names.each do |event_type_name|
+      Tengine::Core::HandlerPath.create!(:event_type_name => event_type_name,
+        :driver_id => self.driver.id, :handler_id => self.id)
+    end
+  end
+
+#   def process_event(event, &block)
+#     @caller = eval("self", block.binding)
+#     matched = match?(event)
+#     if matched
+#       # ハンドラの実行
+#       @caller.__safety_driver__(self.driver) do
+#         @caller.__safety_event__(event) do
+#           @caller.instance_eval(&block)
+#         end
+#       end
+#     end
+#   ensure
+#     @caller = nil
+#   end
+
+  def process_event(event)
+    case self.target_instantiation_key
+    when :instance_method then
+      klass = driver.target_class_name.constantize
+      inst = klass.new
+      inst.instance_variable_set(:@__event__, event)
+      m = inst.method(target_method_name)
+      m.arity == 0 ? m.call : m.call(event)
+    when :static then
+      klass = driver.target_class_name.constantize
+      m = klass.method(target_method_name)
+      m.arity == 0 ? m.call : m.call(event)
+    when :binding then
+      # do nothing
+    else
+      raise Tengine::Core::KernelError, "Unsupported target_instantiation_key: #{self.target_instantiation_key.inspect}"
+    end
+  end
+
+  def fire(event_type_name)
+    @caller.fire(event_type_name)
+  end
+
+  def match?(event)
+    result = filter.blank? ? true : Visitor.new(filter, event, driver.session).visit
+    Tengine.logger.debug("match?(#{event.event_type_name.inspect}) => #{result.inspect}")
+    result
+  end
+
+  # HashとArrayで入れ子になったfilterのツリーをルートから各Leafの方向に辿っていくVisitorです。
+  # 正確にはVisitorパターンではないのですが、似ているのでメタファとしてVisitorとしました。
+  class Visitor
+    def initialize(filter, event, session)
+      @filter = filter
+      @event = event
+      @session = Tengine::Core::SessionWrapper.new(session)
+      @current = @filter
+    end
+
+    def visit
+      Tengine.logger.debug("visiting #{@current.inspect}")
+      send(@current['method'])
+    end
+
+    def backup_current(node)
+      backup = @current
+      @current = node
+      begin
+        return yield
+      ensure
+        @current = backup
+      end
+    end
+
+    def and
+      children = @current["children"]
+      # children.all?{|child| backup_current(child){ visit }} # これだと全てのchildrenについて評価せずfalseがあったら処理を抜けてしまいます。
+      children.map{|child| backup_current(child){ visit }}.all?
+    end
+
+    def find_or_mark_in_session
+      name = @current['pattern'].to_s
+      key = "mark_#{name}"
+      if name == @event.event_type_name
+        unless @session.system_properties[key]
+          @session.system_update(key => true)
+          Tengine.logger.debug("session.system_updated #{@session.system_properties.inspect}")
+        end
+        return true
+      else
+        return @session.system_properties[key]
+      end
+    end
+
+    def match_source_name?
+      pattern = @current['pattern']
+      @event.source_name.include?(pattern)
+    end
+
+  end
+
+
+end

data/lib/tengine/core/handler_path.rb

@@ -0,0 +1,33 @@
+require 'tengine/core'
+
+class Tengine::Core::HandlerPath
+  include Mongoid::Document
+  include Mongoid::Timestamps
+
+  field :event_type_name, :type => String
+  field :handler_id, :type => Object
+
+  belongs_to :driver, :index => true, :class_name => "Tengine::Core::Driver"
+
+  scope(:event_type_name, lambda{|v| where(:event_type_name => v)})
+
+  index([ [:event_type_name, Mongo::ASCENDING], [:_id, Mongo::ASCENDING], ])
+
+  class << self
+    def find_handlers(event_type_name)
+      paths = self.event_type_name(event_type_name).to_a
+      driver_id_to_handler_id = paths.inject({}) do |d, path|
+        d[path.driver_id] ||= []
+        d[path.driver_id] << path.handler_id
+        d
+      end
+      drivers = Tengine::Core::Driver.any_in(:_id => paths.map(&:driver_id)).and(:enabled => true, :version => default_driver_version)
+      drivers.map do |driver|
+        driver.handlers.any_in(:_id => driver_id_to_handler_id[driver.id])
+      end.flatten
+    end
+
+    attr_accessor :default_driver_version
+
+  end
+end

data/lib/tengine/core/heartbeat_watcher.rb

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+require 'daemons'
+require 'eventmachine'
+require 'mongoid'
+require 'uuid'
+require 'active_support/core_ext/hash/keys'
+
+$LOAD_PATH.push File.expand_path("../../../../lib/", __FILE__)
+
+require 'tengine/core'
+require 'tengine/event'
+require 'tengine/mq'
+
+# explicit loading
+require_relative 'config/heartbeat_watcher'
+require_relative 'method_traceable'
+require_relative 'mongoid_fix'
+
+class Tengine::Core::HeartbeatWatcher
+
+  def initialize argv
+    @uuid = UUID.new.generate
+    @config = Tengine::Core::Config::HeartbeatWatcher.parse argv
+    @daemonize_options = {
+      :app_name => 'tengine_heartbeat_watcher',
+      :ARGV => [@config[:action]],
+      :ontop => !@config[:process][:daemon],
+      :multiple => true,
+      :dir_mode => :normal,
+      :dir => File.expand_path(@config[:process][:pid_dir]),
+    }
+    Tengine::Core::MethodTraceable.disabled = !@config[:verbose]
+  rescue Exception
+    puts "[#{$!.class.name}] #{$!.message}\n  " << $!.backtrace.join("\n  ")
+    raise
+  end
+
+  def pid
+    @pid ||= sprintf "process:%s/%d", ENV["MM_SERVER_NAME"], Process.pid
+  end
+
+  def sender
+    @sender ||= Tengine::Event::Sender.new Tengine::Mq::Suite.new(@config[:event_queue])
+  end
+
+  def send_last_event
+    sender.fire "finished.process.hbw.tengine", :key => @uuid, :source_name => pid, :sender_name => pid, :occurred_at => Time.now, :level_key => :info, :keep_connection => true
+    sender.stop
+  end
+
+  def send_periodic_event
+    sender.fire "hbw.heartbeat.tengine", :key => @uuid, :source_name => pid, :sender_name => pid, :occurred_at => Time.now, :level_key => :debug, :keep_connection => true, :retry_count => 0
+  end
+
+  def send_invalidate_event type, e0
+    obj = e0.as_document.symbolize_keys
+    Tengine.logger.info "Heartbeat expiration detected! for #{e0.event_type_name} of #{e0.source_name}: last seen #{e0.occurred_at} (#{(Time.now - e0.occurred_at).to_f} secs before)"
+    obj.delete :_id
+    obj.delete :confirmed
+    obj.delete :updated_at
+    obj.delete :created_at
+    obj.delete :lock_version
+    obj[:event_type_name] = type
+    obj[:level] = Tengine::Event::LEVELS_INV[:error]
+    e1 = Tengine::Event.new obj
+    sender.fire e1, :keep_connection => true
+  end
+
+  def search_for_invalid_heartbeat
+    t = Time.now
+    a = @config[:heartbeat].to_hash.each_pair.map do |e, h|
+      Tengine::Core::Event.where(
+        :event_type_name => "#{e}.heartbeat.tengine",
+        :occurred_at.lte => t - h[:expire]
+        )
+    end
+    a.flatten.each_next_tick do |i|
+      yield i if i
+    end
+  end
+
+  def run(__file__)
+    case @config[:action].to_sym
+    when :start
+      start_daemon(__file__)
+    when :stop
+      stop_daemon(__file__)
+    when :restart
+      stop_daemon(__file__)
+      start_daemon(__file__)
+    end
+  end
+
+  def start_daemon(__file__)
+    pdir = File.expand_path @config[:process][:pid_dir]
+    fname = File.basename __file__
+    cwd = Dir.getwd
+    Daemons.run_proc(fname, @daemonize_options) do
+      Dir.chdir(cwd) { self.start }
+    end
+  end
+
+  def stop_daemon(__file__)
+    fname = File.basename __file__
+    Daemons.run_proc(fname, @daemonize_options)
+  end
+
+  def shutdown
+    EM.run do
+      EM.cancel_timer @invalidate if @invalidate
+      EM.cancel_timer @periodic if @periodic
+      send_last_event
+    end
+  end
+
+  def start
+    @config.setup_loggers
+
+    Mongoid.config.from_hash @config[:db]
+    Mongoid.config.option :persist_in_safe_mode, :default => true
+
+    require 'amqp'
+    Mongoid.logger = AMQP::Session.logger = Tengine.logger
+
+    EM.run do
+      sender.wait_for_connection do
+        @invalidate = EM.add_periodic_timer 1 do # !!! MAGIC NUMBER
+          search_for_invalid_heartbeat do |obj|
+            type = case obj.event_type_name
+                   when /job|core|hbw|resourcew|atd/ then
+                     "expired.#$&.heartbeat.tengine"
+                   end
+            EM.next_tick do
+              send_invalidate_event type, obj
+            end
+          end
+        end
+        int = @config[:heartbeat][:hbw][:interval].to_i
+        if int and int > 0
+          @periodic = EM.add_periodic_timer int do
+            send_periodic_event
+          end
+        end
+      end
+    end
+  end
+
+  extend Tengine::Core::MethodTraceable
+  method_trace(*instance_methods(false))
+end
+
+# 
+# Local Variables:
+# mode: ruby
+# coding: utf-8-unix
+# indent-tabs-mode: nil
+# tab-width: 4
+# ruby-indent-level: 2
+# fill-column: 79
+# default-justification: full
+# End:

data/lib/tengine/core/io_to_logger.rb

@@ -0,0 +1,22 @@
+require 'tengine/core'
+
+# see http://www.ruzee.com/blog/2006/11/redirecting-stdout-to-logger-with-ruby-on-rails
+class Tengine::Core::IoToLogger
+  def initialize(logger, method_to_write = :info)
+    @logger = logger
+    @method_to_write = method_to_write
+  end
+  def puts(str)
+    @logger.send(@method_to_write, str.strip)
+  end
+  def write(str)
+    @logger.send(@method_to_write, str.strip)
+  end
+  alias_method :<<, :puts
+
+  def flush; end # ignore
+
+  alias_method :to_s, :inspect
+
+
+end

data/lib/tengine/core/kernel.rb

@@ -0,0 +1,510 @@
+# -*- coding: utf-8 -*-
+require 'tengine/core'
+
+require 'tengine/event'
+require 'tengine/mq'
+require 'eventmachine'
+require 'selectable_attr'
+
+class Tengine::Core::Kernel
+  include ::SelectableAttr::Base
+  include Tengine::Core::KernelRuntime
+  include Tengine::Core::EventExceptionReportable
+
+  attr_reader :config, :status
+  attr_accessor :before_delegate, :after_delegate
+
+  def initialize(config)
+    @status = :initialized
+    @config = config
+    @processing_event = false
+  end
+
+  def start
+    if block_given?
+      block = Proc.new
+    else
+      block = Proc.new do
+        self.stop
+      end
+    end
+    update_status(:starting)
+    bind
+    if config[:tengined][:wait_activation]
+      update_status(:waiting_activation)
+      wait_for_activation(&block)
+    else
+      activate(&block)
+    end
+  end
+
+  def stop(force = false)
+    if self.status == :running
+      update_status(:shutting_down)
+      mq.initiate_termination do
+        mq.unsubscribe do
+          EM.cancel_timer(@heartbeat_timer) if @heartbeat_timer
+          send_last_event do
+            close_if_shutting_down do
+              update_status(:terminated)
+              EM.stop
+              yield if block_given?
+            end
+          end
+        end
+      end
+    else
+      update_status(:shutting_down)
+      # wait_for_actiontion中の処理を停止させる必要がある
+    end
+  end
+
+  def self.top
+    @top ||= eval("self", TOPLEVEL_BINDING)
+  end
+
+  def dsl_context
+    unless @dsl_context
+      top = self.class.top
+      top.singleton_class.module_eval do
+        include Tengine::Core::DslLoader
+      end
+      top.__kernel__ = self
+      top.config = config
+      @dsl_context = top
+    end
+    @dsl_context
+  end
+  alias_method :context, :dsl_context
+
+  def evaluate
+    dsl_context.__evaluate__
+  end
+
+
+  def bind
+    # dsl_context.__evaluate__
+    # Tengine::Core::stdout_logger.debug("Hanlder bindings:\n" << dsl_context.to_a.inspect)
+    # Tengine::Core::HandlerPath.default_driver_version = config.dsl_version
+    Tengine::Core::HandlerPath.default_driver_version = config.dsl_version
+  end
+
+  def wait_for_activation(&block)
+    activated = false
+    activation_file_name = "#{config[:tengined][:activation_dir]}\/tengined_#{Process.pid}.activation"
+    start_time = Time.now
+    while((Time.now - start_time).to_i <= config[:tengined][:activation_timeout].to_i) do
+      if File.exist?(activation_file_name)
+        # ファイルが見つかった
+        activated = true
+        break
+      end
+      sleep 1
+    end
+    if activated
+      File.delete(activation_file_name)
+      # activate開始
+      activate(&block)
+    else
+      update_status(:shutting_down)
+      raise Tengine::Core::ActivationTimeoutError, "activation file found timeout error."
+    end
+  end
+
+  def activate
+    EM.run do
+      setup_mq_connection
+      subscribe_queue do
+        enable_heartbeat
+        yield(mq) if block_given? # このyieldは接続テストのための処理をTengine::Core:Bootstrapが定義するのに使われます。
+        em_setup_blocks.each{|block| block.call }
+      end
+    end
+  end
+
+  # subscribe to messages in the queue
+  def subscribe_queue
+    confirm = proc do |*|
+      # queueへの接続までできたら稼働中
+      # self.status_key = :running if mq.queue
+      update_status(:running)
+      yield if block_given?
+    end
+    mq.subscribe(:ack => true, :nowait => false, :confirm => confirm) do |headers, msg|
+      process_message(headers, msg)
+    end
+  end
+
+  # @return [true]    メッセージはイベントストアに保存された
+  # @return [それ以外] メッセージは保存されなかった。
+  def process_message(headers, msg)
+    safety_processing_event(headers) do
+      raw_event = parse_event(msg)
+      if raw_event.nil?
+        headers.ack
+        return false
+      end
+      if raw_event.key.blank?
+        Tengine.logger.warn("invalid event which has blank key: #{raw_event.inspect}")
+        headers.ack
+        return
+      end
+
+      delay = ((ENV['TENGINED_EVENT_DEBUG_DELAY'] || '0').to_f || 0.0)
+      sleep delay
+
+      begin
+        # ハートビートは *保存より前に* 特別扱いが必要
+        event = case raw_event.event_type_name
+                when /finished\.process\.([^.]+)\.tengine$/
+                  save_heartbeat_ok(raw_event)
+                when /expired\.([^.]+)\.heartbeat\.tengine$/
+                  save_heartbeat_ng(raw_event)
+                when /heartbeat\.tengine$/ # when の順番に注意
+                  save_heartbeat_beat(raw_event)
+                when /(alert|stop)\.execution\.job\.tengine$/
+                  save_scheduling_event(raw_event)
+                when /\.failed\.tengined$/
+                  save_failed_event(raw_event)
+                else
+                  save_event(raw_event)
+                end
+
+      rescue Mongo::OperationFailure, Mongoid::Errors::Validations => e
+        Tengine.logger.warn("failed to store an event.\n[#{e.class.name}] #{e.message}")
+        # Model.exists?だと上手くいかない時があるのでModel.whereを使っています
+        # fire_failed_event(raw_event) if Tengine::Core::Event.exists?(confitions: { key: raw_event.key, sender_name: raw_event.sender_name })
+        fire_failed_event(raw_event) if Tengine::Core::Event.where(:key => raw_event.key, :sender_name => raw_event.sender_name).count > 0
+        headers.ack
+        return false
+
+      rescue Exception => e
+        Tengine.logger.error("failed to save an event #{raw_event.inspect}\n[#{e.class.name}] #{e.message}")
+        headers.ack
+        return false
+      end
+
+      unless event
+        headers.ack
+        return false
+      end
+      event.kernel = self
+
+      ack_policy = ack_policy_for(event)
+      safety_processing_headers(headers, event, ack_policy) do
+        ack if ack_policy == :at_first
+        handlers = find_handlers(event)
+        safty_handlers(handlers) do
+          delegate(event, handlers)
+          headers.ack if all_submitted?
+        end
+      end
+      close_if_shutting_down
+      true
+    end
+  end
+
+  require 'uuid'
+  HEARTBEAT_EVENT_TYPE_NAME = "core.heartbeat.tengine".freeze
+  HEARTBEAT_ATTRIBUTES = {
+    :key => UUID.new.generate,
+    :level => Tengine::Event::LEVELS_INV[:debug],
+    :source_name => sprintf("process:%s/%d", ENV["MM_SERVER_NAME"], Process.pid),
+    :sender_name => sprintf("process:%s/%d", ENV["MM_SERVER_NAME"], Process.pid),
+    :retry_count => 0,
+  }.freeze
+
+  def enable_heartbeat
+    n = config[:heartbeat][:core][:interval].to_i
+    if n and n > 0
+      EM.defer do
+        @heartbeat_timer = EM.add_periodic_timer(n) do
+          Tengine::Core.stdout_logger.debug("sending heartbeat") if config[:verbose]
+          sender.fire(HEARTBEAT_EVENT_TYPE_NAME, HEARTBEAT_ATTRIBUTES.dup)
+        end
+      end
+    end
+  end
+
+  def fire(*args, &block)
+    sender.fire(*args, &block)
+  end
+
+  def sender
+    unless @sender
+      @sender = Tengine::Event::Sender.new(mq)
+      @sender.default_keep_connection = true
+    end
+    @sender
+  end
+
+  def mq
+    @mq ||= Tengine::Mq::Suite.new(config[:event_queue])
+  end
+
+  private
+
+  def setup_mq_connection
+    mq.add_hook :'connection.on_tcp_connection_failure' do |set|
+      case @status when :terminated, :shutting_down then
+        raise "Could not properly shut down; MQ broker is missing."
+      end
+    end
+    
+    # see http://rdoc.info/github/ruby-amqp/amqp/master/file/docs/ErrorHandling.textile#Recovering_from_network_connection_failures
+    # mq.connection raiases AMQP::TCPConnectionFailed unless connects to MQ.
+    mq.add_hook :'connection.on_error' do |conn, connection_close|
+      Tengine::Core.stderr_logger.error("mq.connection.on_error connection_close: " << connection_close.inspect)
+    end
+    mq.add_hook :'connection.on_tcp_connection_loss' do |conn, settings|
+      Tengine::Core.stderr_logger.warn("mq.connection.on_tcp_connection_loss.")
+    end
+    mq.add_hook :'connection.after_recovery' do |session, settings|
+      Tengine::Core.stderr_logger.info("mq.connection.after_recovery: recovered successfully.")
+    end
+    # on_open, on_closedに渡されたブロックは、何度再接続をしても最初の一度だけしか呼び出されないが、
+    # after_recovery(on_recovery)に渡されたブロックは、再接続の度に呼び出されます。
+    # connection.on_open{ Tengine::Core.stderr_logger.info "mq.connection.on_open first time" }
+    # connection.on_closed{ Tengine::Core.stderr_logger.info  "mq.connection.on_closed first time" }
+
+    mq.add_hook :'channel.on_error' do |ch, channel_close|
+      Tengine::Core.stderr_logger.error("mq.channel.on_error channel_close: " << channel_close.inspect)
+      # raise channel_close.reply_text
+      # channel_close.reuse # channel.on_error時にどのように振る舞うべき?
+    end
+  end
+
+  def parse_event(msg)
+    raw_event = Tengine::Event.parse(msg)
+    Tengine.logger.debug("received an event #{raw_event.inspect}")
+    return raw_event
+  rescue Exception => e
+    Tengine.logger.error("failed to parse a message because of [#{e.class.name}] #{e.message}.\n#{msg}")
+    return nil
+  end
+
+  def fire_failed_event(raw_event)
+    EM.next_tick do
+      # failedということはraw_eventはぶっこわれている。あらゆる仮定は無意味だ。
+      event_attributes = {
+        :level => Tengine::Event::LEVELS_INV[:error],
+        :properties => { :original_event => raw_event }
+      }
+      case etn = raw_event.event_type_name
+      when Tengine::Core::Event::EVENT_TYPE_NAME.format
+        Tengine.logger.debug("sending #{raw_event.event_type_name}.failed.tengined event.") if config[:verbose]
+        sender.fire("#{raw_event.event_type_name}.failed.tengined", event_attributes)
+      else
+        Tengine.logger.debug("sending failed.tengined event.") if config[:verbose]
+        sender.fire("failed.tengined", event_attributes)
+      end
+    end
+  end
+
+  def save_failed_event(raw_event)
+    # これに失敗したときにさらに failed_event を fire してしまうと無限
+    # に fire が続いてしまうので NG.
+    event = Tengine::Core::Event.create!(
+      raw_event.attributes.update(:confirmed => (raw_event.level.to_i <= config.confirmation_threshold)))
+    Tengine.logger.debug("saved an event #{event.inspect}")
+    event
+  rescue Mongo::OperationFailure => e
+    Tengine.logger.error("failed to save an event #{raw_event.inspect}\n[#{e.class.name}] #{e.message}")
+    # FIXME!!
+    # このままではログに埋もれてしまうのでなんとかすべき。
+    # 案1 : root@にメールを投げる
+    # 案2 : プロセスが死ぬ
+    # 案3 : ...
+    return nil
+  end
+
+  # 受信したイベントを登録
+  def save_event(raw_event)
+    event = Tengine::Core::Event.create!(
+      raw_event.attributes.update(:confirmed => (raw_event.level.to_i <= config.confirmation_threshold)))
+    Tengine.logger.debug("saved an event #{event.inspect}")
+    event
+  end
+
+  def save_scheduling_event(raw_event)
+    cond = {
+      :event_type_name => raw_event.event_type_name,
+      :source_name => raw_event.source_name,
+    }
+    event = Tengine::Core::Event.find_or_create_then_update_with_block cond do |event|
+      if event.new_record?
+        event.write_attributes raw_event.attributes
+        event.confirmed = (raw_event.level.to_i <= config.confirmation_threshold)
+      else
+        nil
+      end
+    end
+    case event
+    when FalseClass
+      Tengine.logger.error("failed to save event (after several retries). #{raw_event.inspect}")
+    when NilClass
+      Tengine.logger.debug("this event is duplicated, ignoring now. #{raw_event.inspect}")
+    else
+      Tengine.logger.debug("saved an event #{event.inspect}")
+    end
+    return event
+  end
+
+  def save_heartbeat_beat(raw_event)
+    event = Tengine::Core::Event.find_or_create_then_update_with_block :key => raw_event.key do |event|
+      # beatを保存していいのは、
+      # * 以前にひとつも登録がないとき
+      # * もうbeatが保存されているとき
+      # beatを保存してはいけないのは、
+      # * もうokが保存されているとき
+      # * もうngが保存されているとき
+      if event.new_record? or event.event_type_name == raw_event.event_type_name
+        event.write_attributes raw_event.attributes.update(:confirmed => (raw_event.level.to_i <= config.confirmation_threshold))
+      else
+        nil
+      end
+    end
+    case event
+    when FalseClass
+      Tengine.logger.error("failed to save event (after several retries). #{raw_event.inspect}")
+    when NilClass
+      Tengine.logger.debug("this event is duplicated, ignoring now. #{raw_event.inspect}")
+    else
+      Tengine.logger.debug("saved an event #{event.inspect}")
+    end
+    return event
+  end
+
+  def save_heartbeat_ng(raw_event)
+    event = Tengine::Core::Event.find_or_create_then_update_with_block :key => raw_event.key do |event|
+      # ngを保存していいのは、
+      # * 以前にひとつも登録がないとき
+      # * もうbeatが保存されているとき
+      # * もうokが保存されているとき
+      # ngを保存してはいけないのは、
+      # * もうngが保存されているとき
+      if event.new_record? or event.event_type_name != raw_event.event_type_name
+        event.write_attributes raw_event.attributes.update(:confirmed => (raw_event.level.to_i <= config.confirmation_threshold))
+      else
+        nil
+      end
+    end
+    case event
+    when FalseClass
+      Tengine.logger.error("failed to save event (after several retries). #{raw_event.inspect}")
+    when NilClass
+      Tengine.logger.debug("this event is duplicated, ignoring now. #{raw_event.inspect}")
+    else
+      Tengine.logger.debug("saved an event #{event.inspect}")
+    end
+    return event
+  end
+
+  def save_heartbeat_ok(raw_event)
+    event = Tengine::Core::Event.find_or_create_then_update_with_block :key => raw_event.key do |event|
+      # okを保存していいのは、
+      # * 以前にひとつも登録がないとき
+      # * もうbeatが保存されているとき
+      # okを保存してはいけないのは、
+      # * もうokが保存されているとき
+      # * もうngが保存されているとき
+      beat_type_name = raw_event.event_type_name.sub(/^finished\.process\.(.+?)\.tengine$/, "\\1.heartbeat.tengine")
+      if event.new_record? or event.event_type_name == beat_type_name
+        event.write_attributes raw_event.attributes.update(:confirmed => (raw_event.level.to_i <= config.confirmation_threshold))
+      else
+        nil
+      end
+    end
+    case event
+    when FalseClass
+      Tengine.logger.error("failed to save event (after several retries). #{raw_event.inspect}")
+    when NilClass
+      Tengine.logger.debug("this event is duplicated, ignoring now. #{raw_event.inspect}")
+    else
+      Tengine.logger.debug("saved an event #{event.inspect}")
+    end
+    return event
+  end
+
+  # イベントハンドラの取得
+  def find_handlers(event)
+    handlers = Tengine::Core::HandlerPath.find_handlers(event.event_type_name)
+    Tengine.logger.debug("handlers found for #{event.event_type_name.inspect}: " << handlers.map{|h| "#{h.driver.name} #{h.id.to_s}"}.join(", "))
+    handlers
+  end
+
+  def delegate(event, handlers)
+    before_delegate.call if before_delegate.respond_to?(:call)
+    handlers.each do |handler|
+      safety_handler(handler) do
+        # block = dsl_context.__block_for__(handler)
+        report_on_exception(dsl_context, event) do
+          # handler.process_event(event, &block)
+          if handler.match?(event)
+            handler.process_event(event)
+          end
+        end
+      end
+    end
+    after_delegate.call if after_delegate.respond_to?(:call)
+    ActiveSupport::Dependencies.clear unless config.tengined.cache_drivers
+  end
+
+  def close_if_shutting_down
+    # unsubscribed されている場合は安全な停止を行う
+    # return if mq.queue.default_consumer
+    case status when :shutting_down, :terminated then
+      Tengine::Core.stdout_logger.warn("connection closing...")
+      mq.stop do
+        yield if block_given?
+        EM.stop
+      end
+    end
+  end
+
+  STATUS_LIST = [
+    :initialized,        # 初期化済み
+    :starting,           # 起動中
+    :waiting_activation, # 稼働要求待ち
+    :running,            # 稼働中
+    :shutting_down,      # 停止中
+    :terminated,         # 停止済
+  ].freeze
+
+  # TODO 状態遷移図、状態遷移表に基づいたチェックを入れるべき
+  # https://cacoo.com/diagrams/hwYJGxDuumYsmFzP#EBF87
+  def update_status(status)
+    Tengine::Core.stdout_logger.info("#{self.class.name}#update_status #{@status.inspect} ==================> #{status.inspect}")
+    raise ArgumentError, "Unkown status #{status.inspect}" unless STATUS_LIST.include?(status)
+    @status_filepath ||= File.expand_path("tengined_#{Process.pid}.status", config.status_dir)
+    @status = status
+    File.open(@status_filepath, "w"){|f| f.write(status.to_s)}
+  rescue Exception => e
+    Tengine::Core.stderr_logger.error("#{self.class.name}#update_status failure. [#{e.class.name}] #{e.message}\n  " << e.backtrace.join("\n  "))
+    raise e
+  end
+
+  def send_last_event
+    argh = HEARTBEAT_ATTRIBUTES.dup
+    argh[:level] = Tengine::Event::LEVELS_INV[:info]
+    argh.delete :retry_count # use default
+    sender.fire "finished.process.core.tengine", argh do
+      # 他のデーモンと違ってfinishedをfireしたからといってsender.stopし
+      # てよいとは限らない(裏でまだイベント処理中かも)
+      yield
+    end
+  end
+
+  # 自動でログ出力する
+  extend Tengine::Core::MethodTraceable
+  method_trace(:start, :stop, :bind, :wait_for_activation, :activate,
+    :setup_mq_connection, :subscribe_queue, # :update_status, # update_statusは別途ログ出力します
+    :process_message, :parse_event, :fire_failed_event, :save_event,
+    :find_handlers, :delegate, :close_if_shutting_down, :enable_heartbeat
+    )
+end
+
+
+class Tengine::Core::ActivationTimeoutError < StandardError
+end
+