isono 0.1.0

data/lib/isono/node_modules/job_collector.rb

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    class JobCollector < Base
+
+      initialize_hook do
+        rpc = RpcChannel.new(node)
+
+        app = Rack::DataStore.new(Dispatch.new)
+        
+        rpc.register_endpoint('job-collector', app)
+      end
+
+      terminate_hook do
+      end
+
+      class Dispatch
+        # Register new job
+        def regist
+          params = @req.args[0]
+          params[:node_id]=@req.sender
+          job =Models::JobState.new
+          job.set_fields(params, [:job_id, :parent_job_id, :node_id, :state]).save
+        end
+        
+        def update
+          params = @req.args[0]
+          job = Models::JobState.find(:job_id=>params[:job_id])
+          raise "Unknown or JOB ID: #{params[:job_id]}" if job.nil?
+          job.set_fields(params, [:state, :started_at, :finished_at]).save
+        end
+
+        def call(req, res)
+          @req, @res = req, res
+          raise Rack::UnknownMethodError if @req.command == 'call'
+          m = self.method(@req.command)
+          raise Rack::UnknownMethodError if m.nil?
+
+          ret = m.call
+          @res.response(nil)  if @res.responded?
+        end
+      end
+
+    end
+  end
+end

data/lib/isono/node_modules/job_worker.rb

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+
+require 'statemachine'
+
+module Isono
+  module NodeModules
+    class JobWorker < Base
+      include Logger
+
+      JOB_CTX_KEY=:job_worker_ctx
+      
+      initialize_hook do
+        @thread_pool = ThreadPool.new(10)
+        @active_jobs = {}
+
+        RpcChannel.new(node).register_endpoint("job-stats.#{node.node_id}", proc { |req, res|
+                                                 case res.command
+                                                 when 'get'
+                                                   res.response({ :active_jobs => @active_jobs.map {|j| j.to_hash } })
+                                                 else
+                                                   raise Rack::UnknownMethodError
+                                                 end
+                                               })
+      end
+
+      terminate_hook do
+        @thread_pool.shutdown
+      end
+
+      # Start a new long term job.
+      #
+      # @param [String] parent_id Parent Job ID for new job
+      # @param [Proc,nil] run_cb
+      # @param [Proc,nil] fail_cb
+      # @yield The block used as run_cb
+      # 
+      # @example Simply set run block as yield block and parent job ID.
+      #  run('xxxxxx') {
+      #    # do something
+      #  }
+      # @example Set proc{} to both run and fail block.
+      #  run(proc{
+      #        # do something
+      #      }, proc{
+      #        # do rollback on fail
+      #      })
+      def run(parent_id=nil, run_cb=nil, fail_cb=nil, &blk)
+        if run_cb.is_a?(Proc)
+          job = JobContext.new(run_cb, parent_id)
+          job.fail_cb = fail_cb if fail_cb.is_a?(Proc)
+        elsif blk
+          job = JobContext.new(blk, parent_id)
+        else
+          raise ArgumentError, "callbacks were not set propery"
+        end
+        @active_jobs[job.job_id] = job
+        rpc = RpcChannel.new(node)
+
+        rpc.request('job-collector', 'regist', job.to_hash) { |req|
+          req.oneshot = true
+        }
+        
+        @thread_pool.pass {
+          begin
+            Thread.current[JOB_CTX_KEY]=job
+            job.stm.on_start
+            rpc.request('job-collector', 'update', job.to_hash) { |req|
+              req.oneshot = true
+            }
+            job.run_cb.call
+            job.stm.on_done
+          rescue Exception => e
+            job.stm.on_fail(e)
+            if job.fail_cb
+              job.fail_cb.arity == 1 ? job.fail_cb.call(e) : job.fail_cb.call
+            end
+          ensure
+            Thread.current[JOB_CTX_KEY]=nil
+            EventMachine.schedule {
+              rpc.request('job-collector', 'update', job.to_hash) { |req|
+                req.oneshot = true
+              }
+              @active_jobs.delete(job.job_id)
+            }
+          end
+        }
+        job
+      end
+
+      class JobContext < OpenStruct
+        include Logger
+        attr_reader :stm, :run_cb
+        attr_accessor :fail_cb
+        
+        def initialize(run_cb, parent_id=nil)
+          super({:job_id=>Util.gen_id,
+                  :parent_job_id=> parent_id,
+                  :started_at=>nil,
+                  :finished_at=>nil,
+                })
+
+          @run_cb=run_cb
+          @fail_cb=nil
+
+          @stm = Statemachine.build {
+            startstate :init
+            trans :init,   :on_start,  :running, :on_start
+            trans :running, :on_done, :done, :on_done
+            trans :running, :on_fail, :failed, :on_fail
+            trans :init,   :on_fail, :failed, :on_fail
+          }
+          @stm.context = self
+        end
+
+        def state
+          stm.state
+        end
+        
+        def to_hash
+          @table.dup.merge({:state=>@stm.state})
+        end
+
+        def elapsed_time
+          if finished_at && started_at
+            finished_at - started_at
+          else
+            0
+          end
+        end
+
+        private
+        def on_start
+          self.started_at = Time.now
+          logger.info("Job start #{job_id}")
+        end
+
+        def on_done
+          self.finished_at = Time.now
+          logger.info("Job complete #{job_id}: #{elapsed_time} sec")
+        end
+
+        def on_fail(e)
+          self.finished_at = Time.now
+          logger.error("Job failed #{job_id}: #{e}")
+          logger.error(e)
+        end
+        
+      end
+      
+    end
+  end
+end

data/lib/isono/node_modules/node_collector.rb

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    class NodeCollector < Base
+      include Logger
+
+      config_section do
+        desc "time in second to recognize if the agent is timed out"
+        timeout_sec (60*20).to_f
+        desc "the agent to be killed from the datasource after the time of second"
+        kill_sec (60*20*2).to_f
+        desc ""
+        gc_period 20.0
+      end
+
+      initialize_hook do
+        # GC event trigger for agent timer & status
+        @gc_timer = EM::PeriodicTimer.new(config_section.gc_period) {
+          event = EventChannel.new(self.node)
+          DataStore.pass {
+            # Sqlite3 is unlikely to modify table while iterating
+            # the result set. the following is the case of the
+            # iteration for the opened result set.
+            # Models::AgentPool.dataset.each { |row|
+            # 
+            # while Model.dataset.all, it returns a Ruby array
+            # containing rows so that i had no table lock exception.
+            # see:
+            # http://www.mail-archive.com/sqlite-users@sqlite.org/msg03328.html
+            # TODO: paging support for the large result set.
+            Models::NodeState.dataset.all.each { |row|
+              
+              sm = row.state_machine
+              next if sm.state == :offline
+
+              diff_time = Time.now - row[:last_ping_at]
+              if sm.state != :timeout && diff_time > config_section.timeout_sec
+                sm.on_timeout
+                row.save_changes
+                event.publish('node_collector/timedout', :args=>[row.values])
+              end
+              
+              if diff_time > config_section.kill_sec
+                sm.on_unmonitor
+
+                event.publish('node_collector/killed', :args=>[row.values])
+                row.delete
+              end
+            }
+          }
+        }
+
+        rpc = RpcChannel.new(node)
+        app = Rack::ObjectMethod.new(myinstance)
+        rpc.register_endpoint('node-collector', Rack.build do
+                                use Rack::DataStore
+                                run app
+                              end)
+      end
+      
+      terminate_hook do
+        @gc_timer.cancel
+      end
+
+      def list
+        Models::NodeState.dataset.all.map{|r| r.values }
+      end
+
+      def notify(node_id, boot_token)
+        event = EventChannel.new(node)
+        
+        a = Models::NodeState.find(:node_id=>node_id) || Models::NodeState.new(:node_id=>node_id)
+        a.state_machine.on_ping
+        if a.new?
+          a.boot_token = boot_token
+          a.save
+          event.publish('node_collector/monitored', :args=>[a.values])
+        else
+          a.save_changes
+          #event.publish('node_collector/pong')
+        end
+      end
+
+    end
+  end
+end

data/lib/isono/node_modules/node_heartbeat.rb

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+module Isono
+  module NodeModules
+    class NodeHeartbeat < Base
+
+      config_section do |c|
+        desc "second(s) to wait until send the next heartbeat signal"
+        heartbeat_offset_time 10
+      end
+
+      initialize_hook do
+        @timer = EventMachine::PeriodicTimer.new(config_section.heartbeat_offset_time.to_f) {
+          rpc = RpcChannel.new(node)
+          rpc.request('node-collector', 'notify', manifest.node_id, node.boot_token) do |req|
+            req.oneshot = true
+          end
+        }
+      end
+      
+      terminate_hook do
+        @timer.cancel
+      end
+    end
+  end
+end

data/lib/isono/node_modules/rpc_channel.rb

@@ -0,0 +1,482 @@
+# -*- coding: utf-8 -*-
+
+require 'thread'
+require 'statemachine'
+require 'ostruct'
+
+module Isono
+  module NodeModules
+    class RpcChannel < Base
+      include Logger
+
+      AMQP_EXCHANGE='isono.rpc'
+
+      class RpcError < RuntimeError; end
+      class UnknownEndpointError < RpcError; end
+      class DuplicateEndpointError < RpcError; end
+
+      config_section do
+        desc "default timeout duration in second until receive response."
+        timeout_sec (60*3).to_f
+      end
+
+      initialize_hook do
+        @active_requests = {}
+        @endpoints = {}
+        amq.direct(AMQP_EXCHANGE, {:auto_delete=>true})
+        amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).subscribe { |header, data|
+          event = EventChannel.new(self.node)
+          req = @active_requests[header.message_id]
+          if req
+            data = Serializer.instance.unmarshal(data)
+            req.process_event(:on_received, data)
+            event.publish('rpc/response_received', :args=>[header.message_id])
+            begin
+              case data[:type]
+              when :inprogress
+                req.progress_cb.call(data[:msg]) if req.progress_cb
+              when :error
+                req.process_event(:on_error, data)
+                req.error_cb.call(data[:msg]) if req.error_cb
+              else
+                req.process_event(:on_success, data)
+                req.success_cb.call(data[:msg]) if req.success_cb
+              end
+            rescue => e
+              logger.error(e)
+            ensure
+              if req.state == :done
+                @active_requests.delete req.ticket
+              end
+            end
+          end
+        }
+
+        # RPC endpoint for statistics info of this node.
+        myinstance.register_endpoint("rpc-stats.#{manifest.node_id}", proc { |req, res|
+                                       case req.command
+                                       when 'get'
+                                         res.response({:active_requests => @active_requests.map {|a| a.hash },
+                                                        :endpoints => @endpoints.keys
+                                                      })
+                                       else
+                                         raise Rack::UnknownMethodError
+                                       end
+                                     })
+      end
+
+      terminate_hook do
+        @endpoints.keys.each { |ns|
+          myinstance.unregister_endpoint(ns)
+        }
+        amq.queue("command-recv.#{manifest.node_id}", {:exclusive=>true}).delete
+      end
+
+      # Make a RPC request to an endpoint.
+      # 
+      # @param [String] endpoint
+      # @param [String] command
+      # @param [Array] args
+      # @param [Proc] &blk Block to setup the request context.
+      # @return [RequestContext,any]
+      #
+      # @example create a sync RPC request.
+      #   rpc.request('endpoint1', 'func1', xxxx)
+      # @example call RPC in async mode.
+      #   rpc.request('endpoint1', 'func1', xxxx) { |req|
+      #     req.on_success { |r|
+      #       puts r
+      #     }
+      #     req.on_error { |r|
+      #       puts r
+      #     }
+      #   }
+      #
+      # @example setup request context and do wait().
+      #   Note that callbacks are 
+      #   rpc.request('endpoint1', 'func1', xxxx) { |req|
+      #     # send new attribute
+      #     req.request[:xxxx] = "sdfsdf"
+      #     # returns synchronized RequestContext to block caller.
+      #     req.synchronize
+      #   }.wait # request() get back the altered RequestCotenxt that has wait().
+      #
+      # @example Create async oneshot call. (do not expect response) 
+      #   rpc.request('endpoint1', 'func1') { |req|
+      #     req.oneshot = true
+      #   }
+      def request(endpoint, command, *args, &blk)
+        req = RequestContext.new(endpoint, command, args)
+        # the block is to setup the request context prior to sending.
+        if blk
+          # async 
+          r = blk.call(req)
+          req = r if r.is_a?(RequestContext)
+          if req.oneshot
+            send_request(req)
+          else
+            check_endpoint(endpoint) { |result|
+              if result
+                send_request(req)
+              else
+                e = UnknownEndpointError.new(endpoint)
+                req.error_cb.call(e) if req.error_cb
+              end
+            }
+          end
+
+          req
+        else
+          # sync
+          req = req.synchronize
+          check_endpoint(endpoint) || raise(UnknownEndpointError, endpoint)
+          send_request(req)
+          req.wait
+        end
+      end
+
+      # Register a new RPC endpoint.
+      #
+      # This method works in sync mode if called at non-EM reactor thread.
+      # @param [String] endpoint
+      # @param [has call() method]  app
+      # @param [Hash] opts
+      def register_endpoint(endpoint, app, opts={})
+        raise TypeError unless app.respond_to?(:call)
+        opts = {:exclusive=>true}.merge(opts)
+        @endpoints[endpoint]={:app=>app, :opts=>opts}
+        
+        # create receive queue for new RPC endpoint.
+        endpoint_proc = proc { |header, data|
+
+          data = Serializer.instance.unmarshal(data)
+          event.publish('rpc/request_received', :args=>[header.message_id])
+
+          resctx = if data[:oneshot]
+                     OneshotResponseContext.new(self.node, header)
+                   else
+                     ResponseContext.new(self.node, header)
+                   end
+          begin
+            req = Rack::Request.new({:sender=>header.reply_to['command-recv.'.size..-1],
+                                      :message_id=>header.message_id
+                                    }.merge(data))
+            res = Rack::Response.new(resctx)
+            ret = app.call(req, res)
+          rescue Exception => e
+            logger.error(e)
+            resctx.response(e) unless resctx.responded?
+          end
+        }
+
+        setup_proc = proc {
+          amq.queue(endpoint_queue_name(endpoint), {:exclusive=>false, :auto_delete=>true}).bind(
+                    AMQP_EXCHANGE, {:key=>endpoint_queue_name(endpoint)}
+                    ).subscribe(:ack=>true, &endpoint_proc) 
+          event.publish('rpc/register', :args=>[endpoint])
+        }
+
+        dm = Util::DeferedMsg.new(30)
+        
+        EventMachine.schedule {
+          amq.queue(endpoint_queue_name(endpoint), {:exclusive=>false, :auto_delete=>true}).status { |messages, consumers|
+            if opts[:exclusive]
+              if consumers.to_i == 0
+                setup_proc.call
+                dm.success
+              else
+                dm.error(DuplicateEndpointError.new("Endpoint is already locked: #{endpoint}"))
+              end
+            else
+              setup_proc.call
+              dm.success
+            end
+
+            # expect to raise DuplicateEndpointError if endpoint exists.
+            # ignore the case of success.
+            dm.wait
+          }
+        }
+
+        dm.wait unless EventMachine.reactor_thread?
+      end
+
+      # Unregister endpoint.
+      # 
+      # @param [String] endpoint endpoint name to be removed
+      def unregister_endpoint(endpoint)
+        if @endpoints.delete(endpoint)
+          dm = Util::DeferedMsg.new(30)
+          EventMachine.schedule {
+            amq.queue(endpoint_queue_name(endpoint)).delete
+            event.publish('rpc/unregister', :args=>[endpoint])
+            dm.success
+          }
+          dm.wait unless EventMachine.reactor_thread?
+        end
+      end
+
+      # Check if the endpoint exists.
+      # @param [String] endpoint endpoint name to be checked
+      def check_endpoint(endpoint, &blk)
+        if blk
+        else
+          dm = Util::DeferedMsg.new(30)
+        end
+
+        EventMachine.schedule {
+          amq.queue(endpoint_queue_name(endpoint), {:exclusive=>false, :auto_delete=>true}).status { |messages, consumers|
+            res = consumers.to_i > 0
+            if blk
+              blk.call(res)
+            else
+              dm.success(res)
+            end
+          }
+        }
+        
+        if blk
+        else
+          dm.wait unless EventMachine.reactor_thread?
+        end
+      end
+      
+      private
+      def endpoint_queue_name(ns)
+        "isono.rpc.endpoint.#{ns}"
+      end
+
+      def event
+        @event ||= EventChannel.new(node)
+      end
+
+      # Publish a RPC request asynchronously.
+      # @param [RequestContext] req Request context object to be
+      # sent. If the context's state is not :init, it will fail.
+      def send_request(req)
+        raise TypeError if !req.is_a?(RequestContext)
+        raise "Request context seems to be sent already: #{req.state}" if req.state != :init
+
+        # possible timeout_sec values:
+        # timeout_sec == -1.0 : to be overwritten to the default timeout.
+        # timeout_sec == 0.0  : never be timed out.
+        # timeout_sec > 0.0   : wait for the user set timeout.
+        if req.timeout_sec == -1.0
+          # set default timeout if no one updated the initial value.
+          req.timeout_sec = config_section.timeout_sec
+        end
+        
+        if req.timeout_sec > 0.0
+          # register the timeout hook.
+          req.timer = EventMachine::Timer.new(req.timeout_sec) {
+            @active_requests.delete req.ticket
+            req.error_cb.call(:timeout) if req.error_cb
+          }
+        end
+        
+        req.process_event(:on_ready)
+        
+        EventMachine.schedule {
+          if !req.oneshot
+            @active_requests[req.ticket] = req
+          end
+          
+          amq.direct(AMQP_EXCHANGE).publish(
+                  Serializer.instance.marshal(req.request_hash),
+                  {:message_id => req.ticket,
+                    :key => endpoint_queue_name(req.endpoint),
+                    :reply_to=>"command-recv.#{manifest.node_id}"}
+                  )
+          req.process_event(:on_sent)
+          event.publish('rpc/request_sent', :args=>[req.hash])
+        }
+      end
+
+      class ResponseContext
+        attr_reader :node, :header
+        
+        def initialize(node, header)
+          @responded = false
+          @node = node
+          @header = header
+        end
+
+        def responded?
+          @responded
+        end
+
+        def progress(ret)
+          EM.schedule {
+            publish(:inprogress, ret)
+          }
+        end
+
+        def response(ret)
+          raise "" if @responded
+
+          EM.schedule {
+            @header.ack
+            if ret.is_a? Exception
+              publish(:error, {:message=> ret.message, :error_type => ret.class.to_s})
+            else
+              publish(:success, ret)
+            end
+            EventChannel.new(@node).publish('rpc/response_sent', :args=>[@header.message_id])
+          }
+          @responded = true
+        end
+
+
+        private
+        def publish(type, body)
+          @node.amq.direct('').publish(Serializer.instance.marshal({:type=>type, :msg=>body}),
+                                       {:key=>@header.reply_to,
+                                         :message_id=>@header.message_id}
+                                       )
+        end
+      end
+
+      # Do nothing when the endpoint trys to send back in case of
+      # oneshot request.
+      class OneshotResponseContext < ResponseContext
+        def progress(ret)
+        end
+
+        def response(ret)
+          raise "" if @responded
+
+          EM.schedule {
+            @header.ack
+            EventChannel.new(@node).publish('rpc/response_sent', :args=>[@header.message_id])
+          }
+          @responded = true
+        end
+      end
+
+      class RequestContext < OpenStruct
+        # They are not to be appeared in @table so that won't be inspect().
+        attr_reader :error_cb, :success_cb, :progress_cb
+        attr_accessor :timer
+
+        def initialize(endpoint, command, args)
+          super({:request=>{
+                    :endpoint=> endpoint,
+                    :command => command,
+                    :args => args
+                  },
+                  :endpoint=> endpoint,
+                  :command => command,
+                  :ticket => Util.gen_id,
+                  :timeout_sec => -1.0,
+                  :oneshot => false,
+                  :sent_at => nil,
+                  :completed_at => nil,
+                  :complete_status => nil,
+                })
+          
+          @success_cb = nil
+          @progress_cb = nil
+          @error_cb = nil
+          @timer = nil
+
+          @stm = Statemachine.build {
+            trans :init, :on_ready, :ready
+            trans :ready, :on_sent, :waiting, proc {
+              self.sent_at=Time.now
+              # freeze request hash not to be modified after sending.
+              self.request.freeze
+            }
+            trans :waiting, :on_received, :waiting
+            trans :waiting, :on_error, :done, proc {
+              self.completed_at=Time.now
+              @timer.cancel if @timer
+              self.complete_status = :fail
+            }
+            trans :waiting, :on_success, :done, proc {
+              self.completed_at=Time.now
+              @timer.cancel if @timer
+              self.complete_status = :success
+            }
+          }
+          @stm.context = self
+        end
+
+        def state
+          @stm.state
+        end
+
+        def process_event(ev, *args)
+          @stm.process_event(ev, *args)
+        end
+
+        def elapsed_time
+          self.completed_at.nil? ? nil : (self.completed_at - self.sent_at)
+        end
+
+        def hash
+          # state, sent_at received_at are readonly values so they are
+          # not pushed in @table.
+          @table.dup.merge({:state=>self.state})
+        end
+
+        def request_hash
+          request.merge({:oneshot=>oneshot})
+        end
+
+        def on_success(&blk)
+          raise ArgumentError unless blk
+          @success_cb = blk
+        end
+
+        def on_progress(&blk)
+          raise ArgumentError unless blk
+          @progress_cb = blk
+        end
+
+        def on_error(&blk)
+          raise ArgumentError unless blk
+          @error_cb = blk
+        end
+
+        def synchronize
+          self.extend RequestSynchronize
+          self
+        end
+
+        module RequestSynchronize
+          def self.extended(mod)
+            raise TypeError, "This module is applicable only for RequestContext" unless mod.is_a?(RequestContext)
+            # overwrite callbacks
+            mod.instance_eval {
+              @q = ::Queue.new
+              
+              on_success { |r|
+                @q << [:success, r]
+              }
+              on_error { |r|
+                @q << [:error, r]
+              }
+            }
+          end
+
+          public
+          def wait()
+            raise "response was received already." if state == :done
+            raise "wait() has to be called at outside of the EventMachine's main loop." if EventMachine.reactor_thread?
+            
+            r = @q.deq
+            
+            case r[0]
+            when :success
+              r[1]
+            when :error
+              raise RpcError, r[1]
+            end
+          end
+        end
+        
+      end
+      
+    end
+  end
+end