flapjack 1.6.0 → 2.0.0b1

data/spec/lib/flapjack/notifier_spec.rb

@@ -1,6 +1,137 @@
 require 'spec_helper'
 require 'flapjack/notifier'
 
-describe Flapjack::Notifier do
+describe Flapjack::Notifier, :logger => true do
+
+  let(:redis) { double(::Redis) }
+
+  let(:queue) { double(Flapjack::RecordQueue) }
+
+  it "starts up, runs and shuts down" do
+    config = {'default_contact_timezone' => 'Australia/Broken_Hill'.taint,
+              'email_queue'     => 'email_notifications',
+              'sms_queue'       => 'sms_notifications',
+              'pagerduty_queue' => 'pagerduty_notifications',
+              'jabber_queue'    => 'jabber_notifications'}
+
+    expect(Flapjack::RecordQueue).to receive(:new).with('notifications',
+      Flapjack::Data::Notification).and_return(queue)
+
+    ['email', 'sms', 'pagerduty', 'jabber'].each do |media_transport|
+      expect(Flapjack::RecordQueue).to receive(:new).with("#{media_transport}_notifications",
+        Flapjack::Data::Alert)
+    end
+
+    lock = double(Monitor)
+    expect(lock).to receive(:synchronize).and_yield
+    expect(queue).to receive(:foreach) # assume no messages for now
+    expect(queue).to receive(:wait).and_raise(Flapjack::PikeletStop)
+
+    notifier = Flapjack::Notifier.new(:lock => lock, :config => config)
+
+    expect(redis).to receive(:quit)
+    allow(Flapjack).to receive(:redis).and_return(redis)
+
+    expect { notifier.start }.to raise_error(Flapjack::PikeletStop)
+  end
+
+  # # moved from notification_spec.rb when the code moved
+
+  # let(:event)   { double(Flapjack::Data::Event) }
+
+  # let(:check)       { double(Flapjack::Data::Check) }
+  # let(:check_state) { double(Flapjack::Data::State) }
+
+  # let(:contact) { double(Flapjack::Data::Contact) }
+
+  # let(:timezone) { double('timezone') }
+
+  # it "generates a notification for an event"
+
+  # it "generates messages for contacts" do
+  #   notification = Flapjack::Data::Notification.new(
+  #     :condition_duration    => 16,
+  #     :severity          => 'critical',
+  #     :type              => 'problem',
+  #     :time              => Time.now,
+  #     :duration          => nil
+  #   )
+  #   expect(notification.save).to be_truthy
+  #   expect(notification).to receive(:check).exactly(3).times.and_return(check)
+
+  #   expect(check).to receive(:id).twice.and_return('abcde')
+  #   expect(check).to receive(:name).twice.and_return('example.com:ping')
+
+  #   state = double(Flapjack::Data::CheckOldState)
+  #   expect(state).to receive(:state).exactly(6).times.and_return('critical')
+
+  #   expect(notification).to receive(:state).exactly(12).times.and_return(state)
+
+  #   alerting_checks_1 = double('alerting_checks_1')
+  #   expect(alerting_checks_1).to receive(:exists?).with('abcde').and_return(false)
+  #   expect(alerting_checks_1).to receive(:<<).with(check)
+  #   expect(alerting_checks_1).to receive(:count).and_return(1)
+
+  #   alerting_checks_2 = double('alerting_checks_1')
+  #   expect(alerting_checks_2).to receive(:exists?).with('abcde').and_return(false)
+  #   expect(alerting_checks_2).to receive(:<<).with(check)
+  #   expect(alerting_checks_2).to receive(:count).and_return(1)
+
+  #   medium_1 = double(Flapjack::Data::Medium)
+  #   expect(medium_1).to receive(:transport).and_return('email')
+  #   expect(medium_1).to receive(:address).and_return('abcde@example.com')
+  #   expect(medium_1).to receive(:alerting_checks).exactly(3).times.and_return(alerting_checks_1)
+  #   expect(medium_1).to receive(:clean_alerting_checks).and_return(0)
+  #   expect(medium_1).to receive(:rollup_threshold).exactly(3).times.and_return(10)
+
+  #   medium_2 = double(Flapjack::Data::Medium)
+  #   expect(medium_2).to receive(:transport).and_return('sms')
+  #   expect(medium_2).to receive(:address).and_return('0123456789')
+  #   expect(medium_2).to receive(:alerting_checks).exactly(3).times.and_return(alerting_checks_2)
+  #   expect(medium_2).to receive(:clean_alerting_checks).and_return(0)
+  #   expect(medium_2).to receive(:rollup_threshold).exactly(3).times.and_return(10)
+
+  #   alert_1 = double(Flapjack::Data::Alert)
+  #   expect(alert_1).to receive(:save).and_return(true)
+  #   alert_2 = double(Flapjack::Data::Alert)
+  #   expect(alert_2).to receive(:save).and_return(true)
+
+  #   expect(Flapjack::Data::Alert).to receive(:new).
+  #     with(:rollup => nil, :acknowledgement_duration => nil,
+  #       :state => "critical", :condition_duration => 16,
+  #       :notification_type => 'problem').and_return(alert_1)
+
+  #   expect(Flapjack::Data::Alert).to receive(:new).
+  #     with(:rollup => nil, :acknowledgement_duration => nil,
+  #       :state => "critical", :condition_duration => 16,
+  #       :notification_type => 'problem').and_return(alert_2)
+
+  #   medium_alerts_1 = double('medium_alerts_1')
+  #   expect(medium_alerts_1).to receive(:<<).with(alert_1)
+  #   expect(medium_1).to receive(:alerts).and_return(medium_alerts_1)
+
+  #   medium_alerts_2 = double('medium_alerts_1')
+  #   expect(medium_alerts_2).to receive(:<<).with(alert_2)
+  #   expect(medium_2).to receive(:alerts).and_return(medium_alerts_2)
+
+  #   check_alerts = double('check_alerts_1')
+  #   expect(check_alerts).to receive(:<<).with(alert_1)
+  #   expect(check_alerts).to receive(:<<).with(alert_2)
+  #   expect(check).to receive(:alerts).twice.and_return(check_alerts)
+
+  #   no_rules = double('no_rules', :all => [])
+  #   expect(contact).to receive(:id).and_return('23')
+  #   expect(contact).to receive(:notification_rules).and_return(no_rules)
+  #   all_media = double('all_media', :all => [medium_1, medium_2], :empty? => false)
+  #   expect(all_media).to receive(:each).and_yield(medium_1).
+  #                                   and_yield(medium_2).
+  #                                   and_return([alert_1, alert_2])
+  #   expect(contact).to receive(:media).and_return(all_media)
+
+  #   alerts = notification.alerts([contact], :default_timezone => timezone)
+  #   expect(alerts).not_to be_nil
+  #   expect(alerts.size).to eq(2)
+  #   expect(alerts).to eq([alert_1, alert_2])
+  # end
 
 end

data/spec/lib/flapjack/pikelet_spec.rb

@@ -1,90 +1,151 @@
 require 'spec_helper'
 require 'flapjack/pikelet'
 
-describe Flapjack::Pikelet do
+describe Flapjack::Pikelet, :logger => true do
 
-  let(:config) { double('config') }
-  let(:redis_config) { double('redis_config') }
+  let(:config)        { double('config') }
+  let(:boot_time)     { double(Time) }
 
-  let(:logger) { double(Flapjack::Logger) }
-
-  let(:fiber) { double(Fiber) }
-
-  let(:time) { Time.now }
+  let(:lock)          { double(Monitor) }
+  let(:stop_cond)     { double(MonitorMixin::ConditionVariable) }
+  let(:finished_cond) { double(MonitorMixin::ConditionVariable) }
+  let(:thread)        { double(Thread) }
+  let(:shutdown)      { double(Proc) }
 
   it "creates and starts a processor pikelet" do
-    expect(Flapjack::Logger).to receive(:new).and_return(logger)
-
     expect(config).to receive(:[]).with('logger').and_return(nil)
+    expect(config).to receive(:[]).with('max_runs').and_return(nil)
 
-    fc = double('coordinator')
+    expect(finished_cond).to receive(:signal)
+    expect(lock).to receive(:new_cond).twice.and_return(stop_cond, finished_cond)
+    expect(lock).to receive(:synchronize).and_yield
+    expect(Monitor).to receive(:new).and_return(lock)
 
     processor = double('processor')
     expect(processor).to receive(:start)
-    expect(Flapjack::Processor).to receive(:new).with(:config => config,
-        :redis_config => redis_config, :boot_time => time, :logger => logger, :coordinator => fc).
+    expect(Flapjack::Processor).to receive(:new).
+      with(:lock => lock, :stop_condition => stop_cond, :config => config,
+           :boot_time => boot_time).
       and_return(processor)
 
-    expect(fiber).to receive(:resume)
-    expect(Fiber).to receive(:new).and_yield.and_return(fiber)
+    expect(Thread).to receive(:new).and_yield.and_return(thread)
 
-    pik = Flapjack::Pikelet.create('processor', :config => config,
-      :redis_config => redis_config, :boot_time => time, :coordinator => fc)
-    expect(pik).to be_a(Flapjack::Pikelet::Generic)
-    pik.start
-  end
+    pikelets = Flapjack::Pikelet.create('processor', shutdown, :config => config,
+      :boot_time => boot_time)
+    expect(pikelets).not_to be_nil
+    expect(pikelets.size).to eq(1)
+    pikelet = pikelets.first
+    expect(pikelet).to be_a(Flapjack::Pikelet::Generic)
 
-  it "creates and starts a generic worker gateway" do
-    expect(Flapjack::Logger).to receive(:new).and_return(logger)
+    pikelet.start
+  end
 
+  it "handles an exception from a processor pikelet, and restarts it, then shuts down" do
+    exc = RuntimeError.new
     expect(config).to receive(:[]).with('logger').and_return(nil)
+    expect(config).to receive(:[]).with('max_runs').and_return(2)
 
-    fc = double('coordinator')
+    expect(finished_cond).to receive(:signal)
+    expect(lock).to receive(:new_cond).twice.and_return(stop_cond, finished_cond)
+    expect(lock).to receive(:synchronize).and_yield
+    expect(Monitor).to receive(:new).and_return(lock)
 
-    email = double('email')
-    expect(email).to receive(:start)
-    expect(Flapjack::Gateways::Email).to receive(:new).with(:config => config,
-        :redis_config => redis_config, :boot_time => time, :logger => logger, :coordinator => fc).
-      and_return(email)
+    processor = double('processor')
+    expect(processor).to receive(:start).twice.and_raise(exc)
+    expect(Flapjack::Processor).to receive(:new).
+      with(:lock => lock, :stop_condition => stop_cond, :config => config,
+           :boot_time => boot_time).
+      and_return(processor)
 
-    expect(fiber).to receive(:resume)
-    expect(Fiber).to receive(:new).and_yield.and_return(fiber)
+    expect(Thread).to receive(:new).and_yield.and_return(thread)
 
-    pik = Flapjack::Pikelet.create('email', :config => config,
-      :redis_config => redis_config, :boot_time => time, :coordinator => fc)
-    expect(pik).to be_a(Flapjack::Pikelet::Generic)
-    pik.start
-  end
+   expect(shutdown).to receive(:call)
 
-  it "creates a thin server gateway" do
-    expect(Flapjack::Logger).to receive(:new).and_return(logger)
+    pikelets = Flapjack::Pikelet.create('processor', shutdown,
+      :config => config, :boot_time => boot_time)
+    expect(pikelets).not_to be_nil
+    expect(pikelets.size).to eq(1)
+    pikelet = pikelets.first
+    expect(pikelet).to be_a(Flapjack::Pikelet::Generic)
 
+    pikelet.start
+  end
+
+  it "creates and starts a http server gateway" do
     expect(config).to receive(:[]).with('logger').and_return(nil)
-    expect(config).to receive(:[]).with('bind_address').and_return('0.0.0.0')
+    expect(config).to receive(:[]).with('max_runs').and_return(nil)
+    expect(config).to receive(:[]).with('bind_address').and_return('127.0.0.1')
     expect(config).to receive(:[]).with('port').and_return(7654)
     expect(config).to receive(:[]).with('timeout').and_return(90)
 
+    expect(finished_cond).to receive(:signal)
+    expect(lock).to receive(:new_cond).twice.and_return(stop_cond, finished_cond)
+    expect(lock).to receive(:synchronize).and_yield
+    expect(Monitor).to receive(:new).and_return(lock)
+
+    binder = double('binder')
+    expect(binder).to receive(:add_tcp_listener).with('127.0.0.1', 7654)
+
     server = double('server')
-    expect(server).to receive(:timeout=).with(90)
-    expect(server).to receive(:start)
-    expect(Thin::Server).to receive(:new).
-      with(/^(?:\d{1,3}\.){3}\d{1,3}$/, 7654,
-        Flapjack::Gateways::Web, :signals => false).
+    expect(server).to receive(:binder).and_return(binder)
+    expect(server).to receive(:run).with(false)
+
+    expect(::Puma::Server).to receive(:new).with(Flapjack::Gateways::Web).
       and_return(server)
 
     expect(Flapjack::Gateways::Web).to receive(:instance_variable_set).
       with('@config', config)
+
+    expect(Thread).to receive(:new).and_yield.and_return(thread)
+
+    expect(Flapjack::Gateways::Web).to receive(:start)
+
+    pikelets = Flapjack::Pikelet.create('web', shutdown, :config => config)
+    expect(pikelets).not_to be_nil
+    expect(pikelets.size).to eq(1)
+    pikelet = pikelets.first
+    expect(pikelet).to be_a(Flapjack::Pikelet::HTTP)
+    pikelet.start
+  end
+
+  it "handles an exception from a http server gateway" do
+    exc = RuntimeError.new
+    expect(config).to receive(:[]).with('logger').and_return(nil)
+    expect(config).to receive(:[]).with('max_runs').and_return(nil)
+    expect(config).to receive(:[]).with('bind_address').and_return('127.0.0.1')
+    expect(config).to receive(:[]).with('port').and_return(7654)
+    expect(config).to receive(:[]).with('timeout').and_return(90)
+
+    expect(finished_cond).to receive(:signal)
+    expect(lock).to receive(:new_cond).twice.and_return(stop_cond, finished_cond)
+    expect(lock).to receive(:synchronize).and_yield
+    expect(Monitor).to receive(:new).and_return(lock)
+
+    binder = double('binder')
+    expect(binder).to receive(:add_tcp_listener).with('127.0.0.1', 7654)
+
+    server = double('server')
+    expect(server).to receive(:binder).and_return(binder)
+    expect(server).to receive(:run).with(false).and_raise(exc)
+
+    expect(::Puma::Server).to receive(:new).with(Flapjack::Gateways::Web).
+      and_return(server)
+
     expect(Flapjack::Gateways::Web).to receive(:instance_variable_set).
-      with('@redis_config', redis_config)
-    expect(Flapjack::Gateways::Web).to receive(:instance_variable_set).
-      with('@logger', logger)
+      with('@config', config)
+
+    expect(Thread).to receive(:new).and_yield.and_return(thread)
+
+    expect(shutdown).to receive(:call)
 
     expect(Flapjack::Gateways::Web).to receive(:start)
 
-    pik = Flapjack::Pikelet.create('web', :config => config,
-      :redis_config => redis_config)
-    expect(pik).to be_a(Flapjack::Pikelet::Thin)
-    pik.start
+    pikelets = Flapjack::Pikelet.create('web', shutdown, :config => config)
+    expect(pikelets).not_to be_nil
+    expect(pikelets.size).to eq(1)
+    pikelet = pikelets.first
+    expect(pikelet).to be_a(Flapjack::Pikelet::HTTP)
+    pikelet.start
   end
 
 end

data/spec/lib/flapjack/processor_spec.rb

@@ -7,64 +7,234 @@ describe Flapjack::Processor, :logger => true do
   # NB: this is only testing the public API of the Processor class, which is pretty limited.
   # (initialize, main, stop). Most test coverage for this class comes from the cucumber features.
 
-  let(:config) { double(Flapjack::Configuration) }
+  let(:lock)  { double(Monitor) }
+  let(:redis) { double(Redis) }
+  let(:multi) { double('multi') }
 
-  # TODO this does too much -- split it up
-  it "starts up, runs and shuts down" do
-    t = Time.now.to_i
+  let(:boot_time) { double(Time) }
 
+  before(:each) do
+    allow(Flapjack).to receive(:redis).and_return(redis)
+  end
+
+  def expect_filters
     expect(Flapjack::Filters::Ok).to receive(:new)
     expect(Flapjack::Filters::ScheduledMaintenance).to receive(:new)
     expect(Flapjack::Filters::UnscheduledMaintenance).to receive(:new)
     expect(Flapjack::Filters::Delays).to receive(:new)
     expect(Flapjack::Filters::Acknowledgement).to receive(:new)
+  end
+
+  let(:global_stats)   { double(Flapjack::Data::Statistic) }
+  let(:instance_stats) { double(Flapjack::Data::Statistic) }
+
+  def expect_counters
+    all_global = double('all_global', :all => [global_stats])
+    expect(Flapjack::Data::Statistic).to receive(:intersect).
+      with(:instance_name => 'global').and_return(all_global)
+
+    expect(Flapjack::Data::Statistic).to receive(:new).
+      with(:created_at => boot_time,
+           :all_events => 0, :ok_events => 0,
+           :failure_events => 0, :action_events => 0,
+           :invalid_events => 0, :instance_name => an_instance_of(String)).
+      and_return(instance_stats)
+  end
+
+  def expect_counters_invalid
+    [global_stats, instance_stats].each do |stats|
+      ['all', 'invalid'].each do |event_type|
+        expect(stats).to receive("#{event_type}_events".to_sym).and_return(0)
+        expect(stats).to receive("#{event_type}_events=".to_sym).with(1)
+      end
+    end
+  end
 
-    redis = double('redis')
+  it "starts up, runs and shuts down (archiving, accepted)" do
+    expect_filters
+    expect_counters
 
-    expect(redis).to receive(:hset).with(/^executive_instance:/, "boot_time", anything)
-    expect(redis).to receive(:hget).with('event_counters', 'all').and_return(nil)
-    expect(redis).to receive(:hset).with('event_counters', 'all', 0)
-    expect(redis).to receive(:hget).with('event_counters', 'ok').and_return(nil)
-    expect(redis).to receive(:hset).with('event_counters', 'ok', 0)
-    expect(redis).to receive(:hget).with('event_counters', 'failure').and_return(nil)
-    expect(redis).to receive(:hset).with('event_counters', 'failure', 0)
-    expect(redis).to receive(:hget).with('event_counters', 'action').and_return(nil)
-    expect(redis).to receive(:hset).with('event_counters', 'action', 0)
-    expect(redis).to receive(:hget).with('event_counters', 'invalid').and_return(nil)
-    expect(redis).to receive(:hset).with('event_counters', 'invalid', 0)
+    expect(lock).to receive(:synchronize).and_yield
 
-    expect(redis).to receive(:hset).with(/^event_counters:/, 'all', 0)
-    expect(redis).to receive(:hset).with(/^event_counters:/, 'ok', 0)
-    expect(redis).to receive(:hset).with(/^event_counters:/, 'failure', 0)
-    expect(redis).to receive(:hset).with(/^event_counters:/, 'action', 0)
-    expect(redis).to receive(:hset).with(/^event_counters:/, 'invalid', 0)
+    processor = Flapjack::Processor.new(:lock => lock,
+      :config => {'queue' => 'events', 'archive_events' => true,
+        'events_archive_maxage' => 3000}, :boot_time => boot_time)
 
-    expect(redis).to receive(:expire).with(/^executive_instance:/, anything)
-    expect(redis).to receive(:expire).with(/^event_counters:/, anything).exactly(4).times
+    event_json = double('event_json')
+    event_data = double(event_data)
+    event = double(Flapjack::Data::Event)
 
-    # redis.should_receive(:hincrby).with('event_counters', 'all', 1)
-    # redis.should_receive(:hincrby).with(/^event_counters:/, 'all', 1)
+    expect(redis).to receive(:rpoplpush).with('events', /^events_archive:/).twice.and_return(event_json, nil)
+    expect(redis).to receive(:expire).with(/^events_archive:/, kind_of(Integer))
 
-    expect(Flapjack::Data::Event).to receive(:pending_count).with('events', :redis => redis).and_return(0)
+    expect(Flapjack::Data::Event).to receive(:parse_and_validate).
+      with(event_json).and_return([event_data, []])
+    expect(Flapjack::Data::Event).to receive(:new).with(event_data).and_return(event)
+    expect(redis).to receive(:brpop).with('events_actions').and_raise(Flapjack::PikeletStop)
+    expect(redis).to receive(:quit)
 
-    expect(Flapjack::RedisPool).to receive(:new).and_return(redis)
+    expect(instance_stats).to receive(:save!)
+    expect(instance_stats).to receive(:persisted?).and_return(true)
+    expect(instance_stats).to receive(:destroy)
 
-    fc = double('coordinator')
+    # TODO spec actual functionality
+    expect(processor).to receive(:process_event).with(event)
 
-    executive = Flapjack::Processor.new(:config => {}, :logger => @logger, :coordinator => fc)
+    expect { processor.start }.to raise_error(Flapjack::PikeletStop)
+  end
 
-    noop_evt = double(Flapjack::Data::Event)
-    expect(noop_evt).to receive(:inspect)
-    expect(noop_evt).to receive(:type).and_return('noop')
-    expect(Flapjack::Data::Event).to receive(:next) {
-      executive.instance_variable_set('@should_quit', true)
-      noop_evt
-    }
+  it "starts up, runs and shuts down (archiving, rejected)" do
+    expect_filters
+    expect_counters
+    expect_counters_invalid
 
-    begin
-      executive.start
-    rescue SystemExit
-    end
+    expect(lock).to receive(:synchronize).and_yield
+
+    processor = Flapjack::Processor.new(:lock => lock,
+      :config => {'queue' => 'events', 'archive_events' => true,
+        'events_archive_maxage' => 3000}, :boot_time => boot_time)
+
+    event_json = double('event_json')
+
+    expect(redis).to receive(:rpoplpush).with('events', /^events_archive:/).twice.and_return(event_json, nil)
+    expect(redis).to receive(:multi).and_yield(multi)
+    expect(multi).to receive(:lrem).with(/^events_archive:/, 1, event_json)
+    expect(multi).to receive(:lpush).with(/^events_rejected:/, event_json)
+    expect(multi).to receive(:expire).with(/^events_archive:/, kind_of(Integer))
+
+    expect(Flapjack::Data::Event).to receive(:parse_and_validate).
+      with(event_json).and_return([nil, ["error"]])
+    expect(Flapjack::Data::Event).not_to receive(:new)
+    expect(redis).to receive(:brpop).with('events_actions').and_raise(Flapjack::PikeletStop)
+    expect(redis).to receive(:quit)
+
+    expect(Flapjack::Data::Statistic).to receive(:lock).and_yield
+    expect(global_stats).to receive(:save!)
+    expect(instance_stats).to receive(:save!).twice
+    expect(instance_stats).to receive(:persisted?).and_return(true)
+    expect(instance_stats).to receive(:destroy)
+
+    expect(processor).not_to receive(:process_event)
+
+    expect { processor.start }.to raise_error(Flapjack::PikeletStop)
   end
 
+  it "starts up, runs and shuts down (not archiving, accepted)" do
+    expect_filters
+    expect_counters
+
+    expect(lock).to receive(:synchronize).and_yield
+
+    processor = Flapjack::Processor.new(:lock => lock, :config => {'queue' => 'events'},
+      :boot_time => boot_time)
+
+    event_json = double('event_json')
+    event_data = double(event_data)
+    event = double(Flapjack::Data::Event)
+
+    expect(redis).to receive(:rpop).with('events').twice.and_return(event_json, nil)
+    expect(Flapjack::Data::Event).to receive(:parse_and_validate).
+      with(event_json).and_return([event_data, []])
+    expect(Flapjack::Data::Event).to receive(:new).with(event_data).and_return(event)
+    expect(redis).to receive(:brpop).with('events_actions').and_raise(Flapjack::PikeletStop)
+    expect(redis).to receive(:quit)
+
+    expect(instance_stats).to receive(:save!)
+    expect(instance_stats).to receive(:persisted?).and_return(true)
+    expect(instance_stats).to receive(:destroy)
+
+    # TODO spec actual functionality
+    expect(processor).to receive(:process_event).with(event)
+
+    expect { processor.start }.to raise_error(Flapjack::PikeletStop)
+  end
+
+  it "starts up, runs and shuts down (not archiving, rejected)" do
+    expect_filters
+    expect_counters
+    expect_counters_invalid
+
+    expect(lock).to receive(:synchronize).and_yield
+
+    processor = Flapjack::Processor.new(:lock => lock,
+      :config => {'queue' => 'events'}, :boot_time => boot_time)
+
+    event_json = double('event_json')
+
+    expect(redis).to receive(:rpop).with('events').twice.and_return(event_json, nil)
+    expect(Flapjack::Data::Event).to receive(:parse_and_validate).
+      with(event_json).and_return([nil, ["error"]])
+    expect(Flapjack::Data::Event).not_to receive(:new)
+    expect(redis).to receive(:multi).and_yield(multi)
+    expect(multi).to receive(:lpush).with(/^events_rejected:/, event_json)
+    expect(redis).to receive(:brpop).with('events_actions').and_raise(Flapjack::PikeletStop)
+    expect(redis).to receive(:quit)
+
+    expect(Flapjack::Data::Statistic).to receive(:lock).and_yield
+    expect(global_stats).to receive(:save!)
+    expect(instance_stats).to receive(:save!).twice
+    expect(instance_stats).to receive(:persisted?).and_return(true)
+    expect(instance_stats).to receive(:destroy)
+
+    expect(processor).not_to receive(:process_event)
+
+    expect { processor.start }.to raise_error(Flapjack::PikeletStop)
+  end
+
+  it "starts up, runs and shuts down everything when queue empty (not archiving, accepted)" do
+    expect_filters
+    expect_counters
+
+    expect(lock).to receive(:synchronize).and_yield
+
+    processor = Flapjack::Processor.new(:lock => lock,
+      :config => {'queue' => 'events', 'exit_on_queue_empty' => true},
+      :boot_time => boot_time)
+
+    event_json = double('event_json')
+    event_data = double(event_data)
+    event = double(Flapjack::Data::Event)
+
+    expect(redis).to receive(:rpop).with('events').twice.and_return(event_json, nil)
+    expect(Flapjack::Data::Event).to receive(:parse_and_validate).
+      with(event_json).and_return([event_data, []])
+    expect(Flapjack::Data::Event).to receive(:new).with(event_data).and_return(event)
+    expect(redis).to receive(:quit)
+
+    expect(instance_stats).to receive(:save!)
+    expect(instance_stats).to receive(:persisted?).and_return(true)
+    expect(instance_stats).to receive(:destroy)
+
+    # TODO spec actual functionality
+    expect(processor).to receive(:process_event).with(event)
+
+    expect { processor.start }.to raise_error(Flapjack::GlobalStop)
+  end
+
+  it "rejects invalid event JSON (archiving)" # do
+#     bad_event_json = '{{{'
+#     expect(redis).to receive(:rpoplpush).
+#       with('events', /^events_archive:/).and_return(bad_event_json, nil)
+#     expect(redis).to receive(:multi)
+#     expect(redis).to receive(:lrem).with(/^events_archive:/, 1, bad_event_json)
+#     expect(redis).to receive(:lpush).with(/^events_rejected:/, bad_event_json)
+#     expect(redis).to receive(:exec)
+#     expect(redis).to receive(:expire)
+
+#     Flapjack::Data::Event.foreach_on_queue('events', :archive_events => true) {|result|
+#       expect(result).to be_nil
+#     }
+#   end
+
+#   it "rejects invalid event JSON (not archiving)" do
+#     bad_event_json = '{{{'
+#     expect(redis).to receive(:rpop).with('events').
+#       and_return(bad_event_json, nil)
+#     expect(redis).to receive(:lpush).with(/^events_rejected:/, bad_event_json)
+
+#     Flapjack::Data::Event.foreach_on_queue('events', :archive_events => false) {|result|
+#       expect(result).to be_nil
+#     }
+#   end
+
 end