tengine_job_agent 0.3.17

data/spec/.gitignore

@@ -0,0 +1 @@
+/pid_for_*

data/spec/config/tengine_job_agent.yml.erb

@@ -0,0 +1,17 @@
+---
+timeout: 1
+# log_dir: "log"
+logfile: "<%= %!log/#{File.basename($PROGRAM_NAME)}-#{`hostname`.strip}-#{Process.pid}.log! %>"
+connection:
+  host: 'localhost'
+  port: 5672
+  # vhost:
+  # user:
+  # pass:
+exchange:
+  name: 'tengine_event_exchange'
+  type: 'direct'
+  durable: true
+heartbeat:
+  job:
+    interval: 1

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'simplecov'
+SimpleCov.start if ENV["COVERAGE"]
+require 'rspec'
+require 'tengine_job_agent'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

data/spec/tengine_job_agent/command_utils_spec.rb

@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'tmpdir'
+require 'tempfile'
+
+describe TengineJobAgent::CommandUtils do
+  describe ".included" do
+    it "ClassMethodsを追加" do
+      foo = Class.new do
+        include TengineJobAgent::CommandUtils
+      end
+      foo.singleton_class.ancestors.should include(TengineJobAgent::CommandUtils::ClassMethods)
+    end
+  end
+
+  context "::ClassMethods" do
+    describe "#load_config" do
+      subject { Class.new { include TengineJobAgent::CommandUtils::ClassMethods }.new }
+
+      it "Hashをかえす" do
+        Dir.chdir File.expand_path("../..", __FILE__) do
+          subject.load_config.should be_kind_of(Hash)
+        end
+      end
+
+      it "./tengine_job_agent.ymlを読む" do
+        Dir.mktmpdir do |nam|
+          Dir.chdir nam do
+            File.open("tengine_job_agent.yml", "wb") {|f| f.puts "foo: bar\n" }
+            subject.load_config.should == { "foo" => "bar" }
+          end
+        end
+      end
+
+      it "./config/tengine_job_agent.ymlを読む" do
+        Dir.mktmpdir do |nam|
+          Dir.chdir nam do
+            Dir.mkdir "config"
+            File.open("config/tengine_job_agent.yml", "wb") {|f| f.puts "foo: bar\n" }
+            subject.load_config.should == { "foo" => "bar" }
+          end
+        end
+      end
+
+      it "/etc/tengine_job_agent.ymlを読む" do
+        begin
+          if File.exist? "/etc/tengine_job_agent.yml"
+            obj = YAML.load_file "/etc/tengine_job_agent.yml"
+            subject.load_config.should == obj
+          else
+            File.open("/etc/tengine_job_agent.yml", "wb") {|f| f.puts "foo: bar\n" }
+            subject.load_config.should == { "foo" => "bar" }
+          end
+        rescue Errno::EACCES
+          pending $!.message
+        end
+      end
+    end
+
+    describe "#new_logger" do
+      subject { Class.new { include TengineJobAgent::CommandUtils::ClassMethods }.new }
+      before { subject.stub(:name).and_return("foobar") }
+
+      it "logfileを指定する場合" do
+        Dir.mktmpdir do |nam|
+          Logger.should_receive(:new).with("foo/bar/baz.log")
+          subject.new_logger('logfile' => "foo/bar/baz.log")
+        end
+      end
+
+      it "logfileもlog_dirも指定する場合" do
+        Dir.mktmpdir do |nam|
+          Logger.should_receive(:new).with(/\/foobar-\d+?.log$/)
+          subject.new_logger({})
+        end
+      end
+
+      it "Loggerを返す" do
+        Dir.mktmpdir do |nam|
+          subject.new_logger('log_dir' => nam).should be_kind_of(Logger)
+        end
+      end
+
+      it "引数はディレクトリである" do
+        Tempfile.new("") do |f|
+          subject.new_logger('log_dir' => "nonexistent").should raise_exception(Errno::ENOENT)
+          subject.new_logger('log_dir' => f.path).should raise_exception(Errno::ENOENT)
+        end
+      end
+
+      it "ログファイルは引数のディレクトリの中にできる" do
+        Dir.mktmpdir do |nam|
+          subject.new_logger('log_dir' => nam)
+          nam.should have_at_least(3).files
+        end
+      end
+
+    end
+
+    describe "#process" do
+      subject { Class.new { include TengineJobAgent::CommandUtils } }
+      let(:instance) { mock(subject.new) }
+      before do
+        instance
+        subject.stub(:new).with(anything, anything, anything).and_return(instance)
+        subject.stub(:name).and_return(File.basename(__FILE__, ".rb"))
+      end
+
+      it "インスタンスを生成してprocessを呼ぶ" do
+        instance.should_receive(:process)
+        Dir.chdir File.expand_path("../..", __FILE__) do
+          subject.process
+        end
+      end
+
+      it "失敗するとfalseを返す" do
+        instance.should_receive(:process).and_raise(RuntimeError)
+        Dir.chdir File.expand_path("../..", __FILE__) do
+          subject.process.should == false
+        end
+      end
+    end
+  end
+end

data/spec/tengine_job_agent/run_spec.rb

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+require 'yaml'
+require 'tengine/support/yaml_with_erb'
+require 'rbconfig'
+
+describe TengineJobAgent::Run do
+
+  before do
+    @log_buffer = StringIO.new
+    @logger = Logger.new(@log_buffer)
+    config = YAML.load_file(File.expand_path("../config/tengine_job_agent.yml.erb",
+                                             File.dirname(__FILE__)))
+    @config = config.inject({}) {|r, (k, v)| r.update k.intern => v }
+  end
+
+  subject do
+    Dir.chdir File.expand_path("../..", __FILE__) do
+      TengineJobAgent::Run.new(@logger, %w"scripts/echo_foo.sh", @config)
+    end
+  end
+
+  it { should_not be_nil }
+
+  describe "#process" do
+
+    let(:f) { mock(File.open("/dev/null")) }
+
+    before do
+      subject.stub(:spawn_watchdog).and_return mock(Numeric.new)
+      f
+      File.stub(:open).with(an_instance_of(String), "r").and_yield(f)
+      File.stub(:open).with(an_instance_of(String), "w")
+    end
+
+    context "正常起動" do
+      it "EXIT_SUCCESS" do
+        STDOUT.stub(:puts).with(an_instance_of(String))
+        f.stub(:gets).and_return("0\n")
+        subject.process.should == true
+      end
+    end
+
+    context "spawnできない" do
+      it "EXIT_FAILURE" do      
+        STDERR.stub(:puts) do |arg|
+          arg.should =~ /foo bar/
+        end
+        f.stub(:read).and_return("foo bar")
+        f.stub(:gets).and_return("foo bar")
+        f.stub(:rewind)
+        subject.process.should == false
+      end
+    end
+
+    context "timeout" do
+      it "EXIT_FAILURE" do
+        f.stub(:gets)
+        lambda { subject.process }.should raise_exception(Timeout::Error)
+      end
+    end
+  end
+
+  describe "#spawn_watchdog" do
+    it "tengine_job_agent_watchdogを起動する" do
+      watchdog = File.expand_path("../../bin/tengine_job_agent_watchdog", File.dirname(__FILE__))
+      Process.should_receive(:spawn).with(RbConfig.ruby, watchdog, an_instance_of(String), anything)
+      subject.spawn_watchdog
+    end
+
+    it "pidを返す" do
+      pid = mock(Numeric.new)
+      Process.stub(:spawn).with(RbConfig.ruby, anything, anything, anything).and_return(pid)
+      subject.spawn_watchdog.should == pid
+    end
+
+    it "終了を待たない" do
+      Process.stub(:spawn).with(RbConfig.ruby, anything, anything, anything)
+      Process.should_not_receive :wait
+      Process.should_not_receive :waitpid
+      Process.should_not_receive :waitpid2
+      subject.spawn_watchdog
+    end
+  end
+
+  describe "#initialize" do
+    it "第一引数にlogger" do
+      watchdog = File.expand_path("../../bin/tengine_job_agent_watchdog", File.dirname(__FILE__))
+      Process.should_receive(:spawn).with(RbConfig.ruby, watchdog, an_instance_of(String), anything)
+      subject.spawn_watchdog
+      @log_buffer.string.should_not be_empty
+    end
+
+    it "第二引数は起動するプロセスへの引数の配列" do
+      watchdog = File.expand_path("../../bin/tengine_job_agent_watchdog", File.dirname(__FILE__))
+      Process.should_receive(:spawn).with(RbConfig.ruby, watchdog, an_instance_of(String), "scripts/echo_foo.sh")
+      subject.spawn_watchdog
+    end
+
+    it "第三引数はconfig" do
+      subject.stub(:timeout) do |tim|
+        tim.should == @config[:timeout]
+      end
+    end
+  end
+end

data/spec/tengine_job_agent/scripts/echo_foo.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+echo 'foo'
+exit 0

data/spec/tengine_job_agent/scripts/sleep.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+typeset -i time=$1
+echo "`date` sleep.sh start"
+sleep `expr 1 + $time`
+echo "`date` sleep.sh finish"

data/spec/tengine_job_agent/watchdog_spec.rb

@@ -0,0 +1,307 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'amqp'
+
+require 'timeout'
+
+require 'yaml'
+require 'tengine/support/yaml_with_erb'
+
+describe TengineJobAgent::Watchdog do
+
+  before do
+    @log_buffer = StringIO.new
+    @logger = Logger.new(@log_buffer)
+    @config = YAML.load_file(File.expand_path("../config/tengine_job_agent.yml.erb",
+                                              File.dirname(__FILE__)))
+  end
+
+  subject do
+    @pid_path = "/dev/null"
+    echo_foo = File.expand_path "../scripts/echo_foo.sh", __FILE__
+    TengineJobAgent::Watchdog.new(@logger, [@pid_path, echo_foo], @config)
+  end
+
+  it { should_not be_nil }
+
+  describe "#process" do
+    let(:pid) { mock(Numeric.new) }
+    let(:stat) { mock($?) }
+    before do
+      bigzero = (1 << 1024).coerce(0)[0]
+      pid.stub(:to_int).and_return(bigzero)
+      stat.stub(:exitstatus).and_return(bigzero)
+    end
+
+    it "spawnする" do
+      subject.should_receive(:spawn_process).and_return(pid)
+      subject.stub(:detach_and_wait_process).with(pid).and_return(stat)
+      subject.stub(:fire_finished).with(pid, stat)
+      sender = mock(:sender)
+      sender.stub_chain(:mq_suite, :ensures).with(:connection).and_yield
+      subject.stub(:sender).and_return(sender)
+      sender.stub(:wait_for_connection).and_yield
+      EM.run do
+        EM.add_timer(0.1) { EM.stop }
+        subject.process
+      end
+    end
+
+    it "子プロセスを待つ" do
+      subject.stub(:spawn_process).and_return(pid)
+      subject.should_receive(:detach_and_wait_process).and_return(stat)
+      subject.stub(:fire_finished).with(pid, stat)
+      sender = mock(:sender)
+      sender.stub_chain(:mq_suite, :ensures).with(:connection).and_yield
+      subject.stub(:sender).and_return(sender)
+      sender.stub(:wait_for_connection).and_yield
+      EM.run do
+        EM.add_timer(0.1) { EM.stop }
+        subject.process
+      end
+    end
+
+    context "ファイルへの出力" do
+      before do
+        @echo_foo = File.expand_path "../scripts/echo_foo.sh", __FILE__
+        mock_stdout = mock(:stdout, :path => "/tmp/stdout")
+        mock_stderr = mock(:stderr, :path => "/tmp/stderr")
+        subject.should_receive(:with_tmp_outs).and_yield(mock_stdout, mock_stderr)
+        sender = mock(:sender)
+        sender.stub_chain(:mq_suite, :ensures).with(:connection).and_yield
+        subject.stub(:sender).and_return(sender)
+        sender.stub(:wait_for_connection).and_yield
+      end
+
+      it "実行に成功した場合はPIDが出力される" do
+        subject.should_receive(:spawn_process).and_return(pid)
+        mock_pid_file = mock(:pid_file)
+        File.should_receive(:open).with(@pid_path, "a").and_yield(mock_pid_file)
+        mock_pid_file.should_receive(:puts).with(pid)
+        subject.should_receive(:detach_and_wait_process).with(pid)
+        EM.run do
+          EM.add_timer(0.1) { EM.stop }
+          subject.process
+        end
+      end
+
+      it "ファイルが存在せずspawnに失敗した場合、エラーを出力する" do
+        subject.should_receive(:spawn_process).and_raise(Errno::ENOENT.new(@echo_foo))
+        mock_pid_file = mock(:pid_file)
+        File.should_receive(:open).with(@pid_path, "a").and_yield(mock_pid_file)
+        mock_pid_file.should_receive(:puts).with("[Errno::ENOENT] No such file or directory - #{@echo_foo}")
+        timeout(0.5) do
+          EM.run do
+            # EM.add_timer(0.1) { EM.stop } # 起動に失敗したらEMは自動でstopされる
+            subject.process
+          end
+        end
+      end
+    end
+
+  end
+
+  describe "#spawn_process" do
+    let(:pid) { mock(Numeric.new) }
+    let(:thr) { mock(Thread.start do Thread.stop end) }
+    let(:stat) { mock($?) }
+    before do
+      o = mock(STDOUT)
+      e = mock(STDERR)
+      o.stub(:path).and_return(String.new)
+      e.stub(:path).and_return(String.new)
+      subject.instance_eval do
+        @stdout = o
+        @stderr = e
+      end
+    end
+
+    it "spawnする" do
+      echo_foo = File.expand_path "../scripts/echo_foo.sh", __FILE__
+      Process.should_receive(:spawn).with(echo_foo, an_instance_of(Hash)).and_return(pid)
+      subject.spawn_process.should == pid
+    end
+
+    it "No such file or directoryで失敗する" do
+      echo_foo = File.expand_path "../scripts/echo_foo.sh", __FILE__
+      Process.should_receive(:spawn).with(echo_foo, an_instance_of(Hash)).
+        and_raise(Errno::ENOENT.new(echo_foo))
+      expect {
+        subject.spawn_process
+      }.to raise_error(Errno::ENOENT)
+    end
+  end
+
+  describe "#detach_and_wait_process" do
+    let(:pid) { mock(Numeric.new) }
+    let(:stat) { mock($?) }
+    before do
+      bigzero = (1 << 1024).coerce(0).first
+      stat.stub(:exitstatus).and_return(bigzero)
+      pid.stub(:to_int).and_return(bigzero)
+      Process.stub(:waitpid2).with(pid) do
+        sleep 3
+        [pid, stat]
+      end
+      subject.stub(:fire_finished) do EM.stop end
+      subject.stub(:fire_heartbeat).with(pid).and_yield
+    end
+
+    it "pidを待つ" do
+      EM.run do
+        subject.should_receive(:fire_finished) do EM.stop end
+        subject.detach_and_wait_process(pid)
+      end
+    end
+
+    it "heartbeatをfireしつづける" do
+      EM.run do
+        subject.should_receive(:fire_heartbeat).at_least(2).times.and_yield
+        subject.detach_and_wait_process(pid)
+      end
+    end
+
+    it "https://www.pivotaltracker.com/story/show/21515847" do
+      EM.run do
+        subject.instance_eval { @config["heartbeat"]["job"]["interval"] = 0 }
+        subject.unstub(:fire_heartbeat)
+        subject.should_receive(:fire_heartbeat).at_least(1).times.and_yield
+        subject.detach_and_wait_process(pid)
+      end
+    end
+
+    context "プロセスは正常に動き続けているがfireに失敗した場合" do
+      it "その回のfireはあきらめる。例外などで死なない" do
+        EM.run do
+          subject.unstub(:fire_heartbeat)
+          s = mock(Tengine::Event::Sender.new)
+          subject.stub(:sender).and_return(s)
+          def s.fire e, h, &b
+            h[:retry_count].should_not == nil
+            h[:retry_count].should == 0
+            b.yield if b
+          end
+          expect {
+            subject.detach_and_wait_process(pid)
+          }.to_not raise_exception(Tengine::Event::Sender::RetryError)
+        end
+      end
+    end
+
+    context "finishしたときのtimerの挙動" do
+      def live_timers_count
+        # これはひどい...
+        ObjectSpace.each_object(EM::PeriodicTimer).reject do |i|
+          i.instance_eval do
+            @cancelled
+          end
+        end
+      end
+      it "https://www.pivotaltracker.com/story/show/21466285" do
+        n = live_timers_count
+        EM.run do
+          subject.detach_and_wait_process(pid)
+        end
+        live_timers_count.should == n
+      end
+    end
+  end
+
+  describe "#fire_finished" do
+    let(:pid) { mock(Numeric.new) }
+    let(:stat) { mock($?) }
+    before do
+      pid.stub(:to_int).and_return(-0.0/1.0)
+      conn  = mock(:connection)
+      ch    = Object.new
+
+      AMQP.stub(:connect).with(an_instance_of(Hash)).and_return(conn)
+      AMQP::Channel.stub(:new).with(conn, :prefetch => 1, :auto_recovery => true).and_return(ch)
+      AMQP::Exchange.stub(:new).with(ch, "direct", "exchange1",
+        :passive=>false, :durable=>true, :auto_delete=>false, :internal=>false, :nowait=>true)
+      conn.stub(:on_tcp_connection_loss)
+      conn.stub(:after_recovery)
+      conn.stub(:on_closed)
+      sender = mock(:sender)
+      subject.stub(:sender).and_return(sender)
+      sender.stub(:wait_for_connection).and_yield
+
+      o = mock(STDOUT)
+      e = mock(STDERR)
+      o.stub(:path).and_return(String.new)
+      e.stub(:path).and_return(String.new)
+      subject.instance_eval do
+        @stdout = o
+        @stderr = e
+      end
+    end
+
+    it "finished.process.job.tengineをfire" do
+      EM.run do
+        FileUtils.stub(:cp).with(an_instance_of(String), an_instance_of(String))
+        stat.stub(:exitstatus).and_return(0)
+        s = mock(Tengine::Event::Sender.new)
+        subject.stub(:sender).and_return s
+        s.stub(:stop)
+        s.should_receive(:fire) do |k, v|
+          k.should == "finished.process.job.tengine"
+          v[:level_key].should == :info
+          v[:properties]["pid"].should == pid
+          v[:properties]["exit_status"].should == stat.exitstatus
+        end
+        subject.fire_finished(pid, stat)
+        EM.add_timer(0.1) { EM.stop }
+      end
+    end
+
+    it "プロセスが失敗していた場合" do
+      EM.run do
+        FileUtils.stub(:cp).with(an_instance_of(String), an_instance_of(String))
+        stat.stub(:exitstatus).and_return(256)
+        s = mock(Tengine::Event::Sender.new)
+        subject.stub(:sender).and_return s
+        s.stub(:stop)
+        s.should_receive(:fire) do |k, v|
+          k.should == "finished.process.job.tengine"
+          v[:level_key].should == :error
+          v[:properties][:message].should =~ /^Job process failed./
+        end
+        subject.fire_finished(pid, stat)
+        EM.add_timer(0.1) { EM.stop }
+      end
+    end
+
+    context "プロセスは正常に終了したがfireに失敗した場合" do
+      it "fireできるようになるまでリトライを続ける" do
+        EM.run do
+          stat.stub(:exitstatus).and_return(0)
+          s = mock(Tengine::Event::Sender.new)
+          n = 0
+          subject.stub(:sender).and_return(s)
+          s.stub(:stop)
+          s.stub(:fire).with("finished.process.job.tengine", an_instance_of(Hash)) do |e, h|
+            if h[:retry_count]
+                h[:retry_count].should > 0
+            end
+          end
+          expect {
+            subject.fire_finished(pid, stat)
+          }.to_not raise_exception(Tengine::Event::Sender::RetryError)
+          EM.add_timer(0.1) { EM.stop }
+        end
+      end
+    end
+  end
+
+  describe "#sender" do
+    before do
+      conn = mock(:connection)
+      conn.stub(:on_tcp_connection_loss)
+      conn.stub(:after_recovery)
+      conn.stub(:on_closed)
+      AMQP.stub(:connect).with(an_instance_of(Hash)).and_return(conn)
+    end
+    subject { TengineJobAgent::Watchdog.new(@logger, %w"", @config).sender }
+    it { should be_kind_of(Tengine::Event::Sender) }
+  end
+end