isono 0.1.0 → 0.2.0

data/lib/isono/rack/job.rb

@@ -5,17 +5,9 @@ module Rack
   class Job < Decorator
     include Logger
 
-    # Response class for nothing response.
-    # It is used when the job request type is :submit.
-    class NullResponse < Response
-      def progress(ret)
-      end
-
-      def response(ret)
-      end
-    end
-
     class JobResponse < Response
+      attr_reader :job
+
       # @param [NodeModules::RpcChannel::ResponseContext] ctx
       # @param [NodeModules::JobWorker::JobContext] jobctx
       def initialize(ctx, jobctx)
@@ -30,6 +22,8 @@ module Rack
     end
 
     class JobRequest < Request
+      attr_reader :job
+      
       # @param [Hash] request_hash
       # @param [NodeModules::JobWorker::JobContext] jobctx
       def initialize(request_hash, jobctx)
@@ -44,30 +38,23 @@ module Rack
     end
     
     def call(req, res)
-      orig_res = res
-      case req.r[:job_request_type]
-      when :submit
-        res = NullResponse.new(res.ctx)
+      job = @job_worker.start do |job|
+        job.job_id = req.r[:job_id]
+        job.parent_job_id = req.r[:parent_job_id]
+        job.run_cb = proc {
+          begin
+            @app.call(JobRequest.new(req.r, job), JobResponse.new(res.ctx, job))
+            res.response(nil) unless res.responded?
+          rescue Exception => e
+            res.response(e) unless res.responded?
+            raise e
+          end
+        }
       end
-      
-      job = @job_worker.run(req.r[:parent_job_id]){
-        begin
-          @app.call(JobRequest.new(req.r, job), JobResponse.new(res.ctx, job))
-          res.response(nil) unless res.responded?
-        rescue Exception => e
-          res.response(e) unless res.responded?
-          raise e
-        end
-      }
 
-      case req.r[:job_request_type]
-      when :submit
-        orig_res.response(job.to_hash)
-      else
-        # send job context info back at the first progress message.
-        # following progress messages to be handled as usual.
-        res.progress(job.to_hash)
-      end
+      # send the new job context info back as the first progress message.
+      # the progress messages that follow will be handled as usual.
+      res.progress(job.to_hash)
     end
   end
 end

data/lib/isono/runner/base.rb

@@ -0,0 +1,150 @@
+# -*- coding: utf-8 -*-
+
+require 'optparse'
+require 'digest/sha1'
+require 'etc'
+
+require 'isono'
+require 'isono/amqp_client'
+
+module Isono
+  module Runner
+
+    module Daemonize
+      # Change privileges of the process
+      # to the specified user and group.
+      def self.change_privilege(user, group=user)
+        logger.info("Changing process privilege to #{user}:#{group}")
+        
+        uid, gid = Process.euid, Process.egid
+        target_uid = Etc.getpwnam(user).uid
+        target_gid = Etc.getgrnam(group).gid
+        
+        if uid != target_uid || gid != target_gid
+          # Change process ownership
+          Process.initgroups(user, target_gid)
+          Process::GID.change_privilege(target_gid)
+          Process::UID.change_privilege(target_uid)
+        end
+      rescue Errno::EPERM => e
+        logger.error("Couldn't change user and group to #{user}:#{group}: #{e}")
+      end
+
+      def self.daemonize(log_io=STDOUT)
+        exit if fork
+        srand
+        trap 'SIGHUP', 'DEFAULT'
+        
+        STDIN.reopen('/dev/null')
+        
+        STDOUT.reopen(log_io)
+        STDERR.reopen(log_io)
+      end
+    end
+    
+    class Base
+      include Logger
+      
+      def initialize()
+        @options = {
+          :amqp_server_uri => URI.parse('amqp://guest:guest@localhost/'),
+          :log_file => nil,
+          :pid_file => nil,
+          :daemonize => false,
+          :config_path => nil,
+          :manifest_path => nil,
+        }
+      end
+
+      def optparse(args)
+        optparse = OptionParser.new do |opts|
+          opts.banner = "Usage: #{$0} [options]"
+          
+          opts.separator ""
+          opts.separator "Options:"
+          opts.on( "-i", "--id ID", "Manually specify the Agent ID" ) {|str| @options[:node_id] = str }
+          opts.on( "-p", "--pid PIDFILE", "pid file path" ) {|str| @options[:pid_file] = str }
+          opts.on( "--log LOGFILE", "log file path" ) {|str| @options[:log_file] = str }
+          opts.on( "--config CONFFILE", "config file path" ) {|str| @options[:config_file] = str }
+          opts.on( "-s", "--server AMQP_URI", "amqp broker server to connect" ) {|str|
+            begin
+              @options[:amqp_server_uri] = URI.parse(str)
+            rescue URI::InvalidURIError => e
+              abort "#{e}"
+            end
+          }
+          opts.on("-b", "Run in background" ) { @options[:daemonize] = false }
+        end
+
+        optparse.parse!(args)
+      end
+      
+      def run(manifest=nil, opts={}, &blk)
+        @options = @options.merge(opts)
+        optparse(ARGV.dup)
+        
+        if manifest.is_a?(String)
+          # load manifest file
+          manifest = Manifest.load_file(manifest)
+        elsif manifest.nil? && @options[:manifest_path]
+          manifest = Manifest.load_file(@options[:manifest_path])
+        end
+
+        if @options[:node_id]
+          # force overwrite node_id if the command line arg was given.
+          manifest.node_instance_id(@options[:node_id])
+        elsif manifest.node_id.nil?
+          # nobody specified the node_id then set the ID in the
+          # default manner. 
+          manifest.node_id(default_node_id)
+        end
+
+        @options[:log_file] ||= "/var/log/%s.log" % [manifest.node_name]
+        @options[:pid_file] ||= "/var/run/%s.pid" % [manifest.node_name]
+
+        if @options[:daemonize]
+          if @options[:log_file]
+            logio = File.open(@options[:log_file], "a")
+          end
+          Daemonize.daemonize(logio || STDOUT)
+        end
+
+        # EM's reactor is shutdown already when EXIT signal is
+        # caught. so that set handler for TERM, INT
+        %w(TERM INT).each { |i|
+          Signal.trap(i) {
+            if @node
+              # force the block to push next loop.
+              # EM.schedule gets the current thread stucked.
+              EventMachine.next_tick {
+                @node.close { EventMachine.stop }
+                @node = nil
+              }
+            end
+          }
+        }
+        Signal.trap(:EXIT) { remove_pidfile if @options[:daemonize] }
+
+        EventMachine.epoll
+        EventMachine.run {
+          @node = run_main(manifest, &blk)
+          raise "run_main() must return Isono::Node object: #{@node.class}" unless @node.is_a?(Isono::Node)
+        }
+      end
+
+      protected
+      def run_main(manifest, &blk)
+        Isono::Node.new(manifest).connect(@options[:amqp_server_uri]) { |n| 
+          @node = n
+          self.instance_eval &blk
+        }
+      end
+      
+      def default_node_id
+        raise NotImplementedError
+      end
+
+    end
+
+  end
+end

data/lib/isono/runner/cli.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+require 'isono'
+
+module Isono
+  module Runner
+    module CLI
+      
+      def run_cli(manifest=nil, &blk)
+        Oneshot.new.run(manifest, &blk)
+      end
+      module_function :run_cli
+
+      class Oneshot < Runner::Base
+        include Logger
+        
+        def run_main(manifest, &blk)
+          Isono::Node.new(manifest).connect(@options[:amqp_server_uri]) do |n|
+            exit(1) unless n.connected?
+            @node = n
+            self.instance_eval &blk
+          end
+        end
+      end
+      
+    end
+  end
+end

data/lib/isono/runner/rpc_server.rb

@@ -54,8 +54,10 @@ module Isono
           def job(command, run_cb=nil, fail_cb=nil, &blk)
             app = if run_cb.is_a?(Proc)
                     proc {
-                      request.fail_cb do
-                        self.instance_eval(&fail_cb)
+                      if fail_cb.is_a?(Proc)
+                        response.fail_cb do
+                          self.instance_eval(&fail_cb) 
+                        end
                       end
 
                       self.instance_eval(&run_cb)
@@ -131,66 +133,32 @@ module Isono
         load_module NodeModules::JobChannel
       end
       
-      def start(manifest=nil, &blk)
-        rpcsvr = Server.new(ARGV)
-        rpcsvr.run(manifest, &blk)
+      def start(manifest=nil, opts={}, &blk)
+        rpcsvr = Server.new(blk)
+        rpcsvr.run(manifest)
       end
       module_function :start
 
-      class Server
-        def initialize(argv)
-          @argv = argv.dup
-          
-          @options = {
-            :amqp_server_uri => URI.parse('amqp://guest:guest@localhost/'),
-          }
-          
-          parser.parse! @argv
-        end
-        
-        def parser
-          @parser ||= OptionParser.new do |opts|
-            opts.banner = "Usage: agent [options]"
-            
-            opts.separator ""
-            opts.separator "Agent options:"
-            opts.on( "-i", "--id ID", "Manually specify the Node ID" ) {|str| @options[:node_id] = str }
-            opts.on( "-s", "--server AMQP_URI", "amqp broker server to connect" ) {|str|
-              begin
-                @options[:amqp_server_uri] = URI.parse(str)
-              rescue URI::InvalidURIError => e
-                abort "#{e}"
-              end
-            }
-          end
-        end
-        
-        def run(manifest=nil, &blk)
-          %w(EXIT).each { |i|
-            Signal.trap(i) { Isono::Node.stop }
-          }
-          
-          # .to_s to avoid nil -> String conversion failure.
-          if @options[:node_id]
-            manifest.node_instance_id(@options[:node_id])
-          elsif manifest.node_instance_id.nil?
-            abort("[ERROR]: manifest.node_istance_id is not set")
-          end
-          
-          EventMachine.epoll
-          EventMachine.run {
-            @node = Isono::Node.new(manifest)
-            @node.connect(@options[:amqp_server_uri], @options) do
-              self.instance_eval(&blk) if blk
-            end
-          }
+      class Server < Base
+        def initialize(builder_block)
+          super()
+          @builder_block = builder_block
         end
-        
+
+        # DSL method
         def endpoint(endpoint, builder)
           raise TypeError unless builder.respond_to?(:build)
           builder.build(endpoint, @node)
         end
         
+        protected
+        def run_main(manifest=nil)
+          @node = Isono::Node.new(manifest)
+          @node.connect(@options[:amqp_server_uri]) do
+            self.instance_eval(&@builder_block) if @builder_block
+          end
+        end
+        
       end
       
     end

data/lib/isono/thread_pool.rb

@@ -18,25 +18,29 @@ module Isono
       @worker_threads = {}
       worker_num.times {
         t = Thread.new {
-          loop {
-            begin
-              while op = @queue.pop
-                if @queue.size > @opts[:stucked_queue_num] && Time.now - @last_stuck_warn_at > 5.0
-                  logger.warn("too many stacked workers: #{@queue.size}")
-                  @last_stuck_warn_at = Time.now
-                end
-                op.call
+          begin
+            while op = @queue.pop
+              if @queue.size > @opts[:stucked_queue_num] && Time.now - @last_stuck_warn_at > 5.0
+                logger.warn("too many stacked jobs: #{@queue.size}")
+                @last_stuck_warn_at = Time.now
               end
-            rescue WorkerTerminateError
-              # someone indicated to terminate this thread
-              # exit from the current loop
-              break
-            rescue Exception => e
-              self.logger.error(e)
+              
+              op.call
             end
-          }
-          @worker_threads.delete(Thread.current.__id__)
-          logger.info("#{Thread.current} is being terminated")
+          rescue WorkerTerminateError
+            # someone indicated to terminate this thread
+            # exit from the current loop
+            break
+          rescue Exception => e
+            logger.error(e)
+            retry
+          ensure
+            EM.schedule {
+              @worker_threads.delete(Thread.current.__id__)
+              logger.debug("#{Thread.current} is being terminated")
+            }
+          end
+          
         }
         @worker_threads[t.__id__] = t
       }
@@ -100,8 +104,9 @@ module Isono
 
     # Immediatly shutdown all the worker threads
     def shutdown()
-      @worker_threads.each {|t|
-        t.raise WorkerTerminateError
+      @worker_threads.each {|id, t|
+        t.__send__(:raise, WorkerTerminateError)
+        Thread.pass
       }
     end
 

data/lib/isono/util.rb

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 require 'digest/sha1'
-require 'hmac-sha1'
 require 'thread'
 require 'stringio'
 
@@ -44,7 +43,7 @@ module Isono
            when 'Linux'
              `/sbin/ip route get 8.8.8.8`.split("\n")[0].split.last
            when 'SunOS'
-             `/sbin/ifconfig $(route get 1.1.1.1  | awk '$1 == "interface:" {print $2}') | awk '$1 == "inet" { print $2 }'`
+             `/sbin/ifconfig $(/usr/sbin/route -n get 1.1.1.1  | awk '$1 == "interface:" {print $2}') | awk '$1 == "inet" { print $2 }'`
            else
              raise "Unsupported platform to detect gateway IP address: #{`/bin/uname`}"
            end
@@ -171,6 +170,7 @@ module Isono
         super()
         @thread_wait = th
         @timer_sig = EventMachine.add_timer(timeout) {
+          @on_timeout_hook.call if @on_timeout_hook
           error(TimeoutError.new)
         }
       end
@@ -180,13 +180,21 @@ module Isono
         @thread_called = Thread.current
       end
 
-
       def error(ex)
         raise TypeError unless ex.is_a?(Exception)
         self.enq(ex)
         @thread_called = Thread.current
       end
 
+      def on_timeout(&blk)
+        @on_timeout_hook = blk
+      end
+
+      def cancel
+        EventMachine.cancel_timer(@timer_sig) rescue nil
+        @on_timeout_hook = nil
+      end
+
       def wait
         if (@thread_called == @thread_wait || @thread_called == Thread.current ) &&
             self.empty?
@@ -203,7 +211,7 @@ module Isono
         end
       ensure
         @thread_wait = nil
-        EventMachine.cancel_timer(@timer_sig) rescue nil
+        self.cancel
       end
     end