rflow 0.0.5 → 1.0.0a1

data/spec/rflow_component_port_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper.rb'
+
+describe RFlow::Component::Port do
+  it "should not be connected" do
+    described_class.new.connected?.should be_false
+  end
+end
+
+describe RFlow::Component::HashPort do
+  it "should not be connected" do
+    port_config = double('Port Config')
+    port_config.should_receive(:name).and_return('port')
+    port_config.should_receive(:uuid).and_return('1')
+
+    port = described_class.new(port_config)
+    port.connected?.should be_false
+  end
+end
+
+describe RFlow::Component::InputPort do
+  context ".connect!" do
+    it "should be connected" do
+      connection_double = double('connection')
+      connection_double.should_receive(:connect_input!)
+
+      port_config = double('Port Config')
+      port_config.should_receive(:name).and_return('port')
+      port_config.should_receive(:uuid).and_return('1')
+
+      port = described_class.new(port_config)
+      port.add_connection(nil, connection_double)
+
+      port.connected?.should be_false
+      port.connect!
+      port.connected?.should be_true
+    end
+  end
+end
+
+describe RFlow::Component::OutputPort do
+  context ".connect!" do
+    it "shouldbe connected" do
+      connection_double = double('connection')
+      connection_double.should_receive(:connect_output!)
+
+      port_config = double('Port Config')
+      port_config.should_receive(:name).and_return('port')
+      port_config.should_receive(:uuid).and_return('1')
+
+      port = described_class.new(port_config)
+      port.add_connection(nil, connection_double)
+
+      port.connected?.should be_false
+      port.connect!
+      port.connected?.should be_true
+    end
+  end
+end

data/spec/rflow_configuration_ruby_dsl_spec.rb

@@ -0,0 +1,148 @@
+require 'spec_helper.rb'
+require 'rflow/configuration'
+
+describe RFlow::Configuration::RubyDSL do
+  before(:each) do
+    ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+    RFlow::Configuration.migrate_database
+  end
+
+  it "should correctly process an empty DSL" do
+    described_class.configure { |c| }
+
+    config = RFlow::Configuration.new
+    RFlow::Configuration::Shard.count.should == 1
+    RFlow::Configuration::Component.count.should == 0
+    RFlow::Configuration::Port.count.should == 0
+    RFlow::Configuration::Connection.count.should == 0
+
+    puts config.to_s
+  end
+
+  it "should correctly process a component declaration" do
+    described_class.configure do |c|
+      c.component 'boom', 'town', 'opt1' => 'OPT1', 'opt2' => 'OPT2'
+    end
+
+    config = RFlow::Configuration.new
+    RFlow::Configuration::Shard.count.should == 1
+    RFlow::Configuration::Component.count.should == 1
+    RFlow::Configuration::Port.count.should == 0
+    RFlow::Configuration::Connection.count.should == 0
+
+    component = RFlow::Configuration::Component.all.first
+    component.name.should == 'boom'
+    component.specification.should == 'town'
+    component.options.should == {'opt1' => 'OPT1', 'opt2' => 'OPT2'}
+  end
+
+  it "should correctly process a connect declaration" do
+    described_class.configure do |c|
+      c.component 'first', 'First'
+      c.component 'second', 'Second'
+      c.connect 'first#out' => 'second#in'
+      c.connect 'first#out' => 'second#in[inkey]'
+      c.connect 'first#out[outkey]' => 'second#in'
+      c.connect 'first#out[outkey]' => 'second#in[inkey]'
+    end
+
+    config = RFlow::Configuration.new
+    RFlow::Configuration::Shard.count.should == 1
+    RFlow::Configuration::Component.count.should == 2
+    RFlow::Configuration::Port.count.should == 2
+    RFlow::Configuration::Connection.count.should == 4
+
+    first_component = RFlow::Configuration::Component.where(name: 'first').first
+    second_component = RFlow::Configuration::Component.where(name: 'second').first
+
+    first_component.specification.should == 'First'
+    first_component.input_ports.count.should == 0
+    first_component.output_ports.count.should == 1
+    first_component.output_ports.first.name.should == 'out'
+    first_connections = first_component.output_ports.first.connections.all
+    first_connections.count.should == 4
+    first_connections[0].input_port_key.should be_nil
+    first_connections[0].output_port_key.should be_nil
+    first_connections[1].input_port_key.should == 'inkey'
+    first_connections[1].output_port_key.should be_nil
+    first_connections[2].input_port_key.should be_nil
+    first_connections[2].output_port_key.should == 'outkey'
+    first_connections[3].input_port_key.should == 'inkey'
+    first_connections[3].output_port_key.should == 'outkey'
+
+    second_component.specification.should == 'Second'
+    second_component.input_ports.count.should == 1
+    second_component.output_ports.count.should == 0
+    second_component.input_ports.first.name.should == 'in'
+    second_connections = second_component.input_ports.first.connections.all
+    second_connections.count.should == 4
+
+    first_connections.should == second_connections
+
+    puts config.to_s
+  end
+
+  it "should correctly process shard declarations" do
+    described_class.configure do |c|
+      c.component 'first', 'First', :opt1 => 'opt1'
+
+      c.shard "s1", :process => 2 do |s|
+        s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+      end
+
+      c.shard "s2", :type => :process, :count => 10 do |s|
+        s.component 'third', 'Third'
+        s.component 'fourth', 'Fourth'
+      end
+
+      c.component 'fifth', 'Fifth'
+
+      c.connect 'first#out' => 'second#in'
+      c.connect 'second#out[outkey]' => 'third#in[inkey]'
+      c.connect 'second#out' => 'third#in2'
+      c.connect 'third#out' => 'fourth#in'
+      c.connect 'third#out' => 'fifth#in'
+    end
+
+    config = RFlow::Configuration.new
+    RFlow::Configuration::Shard.count.should == 3
+    RFlow::Configuration::Component.count.should == 5
+    RFlow::Configuration::Port.count.should == 8
+    RFlow::Configuration::Connection.count.should == 5
+
+    shards = RFlow::Configuration::Shard.all
+    shards.map(&:name).should == ['DEFAULT', 's1', 's2']
+    shards.first.components.all.map(&:name).should == ['first', 'fifth']
+    shards.second.components.all.map(&:name).should == ['second']
+    shards.third.components.all.map(&:name).should == ['third', 'fourth']
+
+    RFlow::Configuration::Port.all.map(&:name).should == ['out', 'in', 'out', 'in', 'in2', 'out', 'in', 'in']
+
+    RFlow::Configuration::Connection.all.map(&:name).should == ['first#out=>second#in',
+                                                                'second#out[outkey]=>third#in[inkey]',
+                                                                'second#out=>third#in2',
+                                                                'third#out=>fourth#in',
+                                                                'third#out=>fifth#in']
+
+    puts config.to_s
+  end
+
+  it "should not allow two components with the same name" do
+    expect do
+      described_class.configure do |c|
+        c.component 'first', 'First'
+        c.component 'first', 'First'
+      end
+    end.to raise_error
+  end
+
+  it "should not allow two shards with the same name" do
+    expect do
+      described_class.configure do |c|
+        c.shard("s1", :process => 2) { |s| }
+        c.shard("s1", :process => 2) { |s| }
+      end
+    end.to raise_error
+  end
+
+end

data/spec/rflow_configuration_spec.rb

@@ -9,7 +9,7 @@ describe RFlow::Configuration do
 #    RFlow::Configuration.available_data_extensions.clear
   end
 
-  
+
   describe '.add_available_data_type' do
     context 'if passed a data_serialization that is not avro or xml' do
       it "should throw an exception" do
@@ -27,7 +27,7 @@ describe RFlow::Configuration do
   end
 
   describe "Data Extensions" do
-  
+
     describe ".add_available_data_extension" do
       context 'if passed a non-module data extension' do
         it "should throw an exception" do
@@ -36,7 +36,7 @@ describe RFlow::Configuration do
           end.to raise_error(ArgumentError)
         end
       end
-      
+
       context "if passed a valid Module as a data extension" do
         it "should update the available_data_extensions" do
           num_extensions = RFlow::Configuration.available_data_extensions['data_type'].size
@@ -47,7 +47,7 @@ describe RFlow::Configuration do
         end
       end
     end
-    
+
     it "should perform simple 'prefix'-based inheritance for extensions" do
       RFlow::Configuration.add_available_data_extension('A', A = Module.new)
       RFlow::Configuration.add_available_data_extension('A::B', B = Module.new)

data/spec/rflow_message_data_raw.rb

@@ -3,14 +3,14 @@ require 'spec_helper.rb'
 require 'rflow/components/raw'
 
 describe 'RFlow::Message::Data::Raw Avro Schema' do
-  before(:each) do 
+  before(:each) do
     @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
   end
 
   it "should load the schema" do
     @schema_string.should_not == nil
   end
-  
+
   it "should encode and decode an object" do
     raw = {'raw' => 'rawdata'}
 

data/spec/rflow_message_data_spec.rb

@@ -17,22 +17,22 @@ describe RFlow::Message::Data do
   end
 
   context "if created with an invalid schema for the serialization" do
-    it "should throw and exception" do 
+    it "should throw and exception" do
       expect {RFlow::Message::Data.new(@invalid_avro_schema_string)}.to raise_error(ArgumentError)
       expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
       expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
     end
   end
-  
+
   context "if created with a valid avro schema and serialization" do
   end
 
-  context "if created with a valid avro schema" do 
-    it "should instantiate correctly" do 
+  context "if created with a valid avro schema" do
+    it "should instantiate correctly" do
       expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
       expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
     end
-    
+
     context "if created with a non-avro data serialization" do
       it "should throw an exception" do
         expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'unknown')}.to raise_error(ArgumentError)
@@ -41,7 +41,7 @@ describe RFlow::Message::Data do
         expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :xml)}.to raise_error(ArgumentError)
       end
     end
-    
+
     context "if created with an avro serialization" do
       it "should instantiate correctly" do
         expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error

data/spec/rflow_message_spec.rb

@@ -50,7 +50,7 @@ describe RFlow::Message do
               @string = 'this is a string to be serialized'
               @avro_serialized_string = encode_avro(@avro_string_schema_string, @string)
             end
-            
+
             it "should instantiate correctly" do
               expect {RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)}.to_not raise_error
               message = RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)
@@ -65,12 +65,12 @@ describe RFlow::Message do
         @invalid_processing_event_hash = {'started_at' => 'bad time string'}
         @invalid_provenance = [@invalid_processing_event_hash]
       end
-      
+
       it "should throw an exception" do
         expect {RFlow::Message.new('string_type', @invalid_provenance)}.to raise_error(ArgumentError)
       end
     end
-    
+
     context "if created with valid provenance" do
       before(:all) do
         @valid_xmlschema_time = '2001-01-01T01:01:01.000001Z'
@@ -87,7 +87,7 @@ describe RFlow::Message do
                                     {"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>@valid_xmlschema_time, "context"=>"context"},
                                    ]
       end
-      
+
       it "should instantiate correctly" do
         p @valid_provenance
         expect {RFlow::Message.new('string_type', @valid_provenance)}.to_not raise_error
@@ -105,7 +105,7 @@ describe RFlow::Message do
         message = RFlow::Message.new('string_type', @valid_provenance)
         message.provenance.map(&:to_hash).should == @valid_provenance_hashes
       end
-      
+
     end
 
     context "if correctly created" do
@@ -119,7 +119,7 @@ describe RFlow::Message do
         message.data.data_object.should == processed_message.data.data_object
       end
     end
-    
+
     context "if data extensions exist" do
       it "should extend the data element with the extension" do
         module ExtensionModule; def ext_method; end; end
@@ -138,9 +138,9 @@ describe RFlow::Message do
   it "should correctly handle large raw types" do
     message = RFlow::Message.new('RFlow::Message::Data::Raw')
     message.data.raw = Array.new(101) { rand(256) }.pack('c*')
-    
+
     message_avro = message.to_avro.force_encoding('BINARY')
-    
+
     processed_message = RFlow::Message.from_avro(message_avro)
     processed_message_avro = processed_message.to_avro.force_encoding('BINARY')
 
@@ -148,13 +148,13 @@ describe RFlow::Message do
 
     encode_avro(@raw_schema, message.data.data_object).should == message.data.to_avro
     decode_avro(@raw_schema, message.data.to_avro).should == message.data.data_object
-    
+
     p message.data.raw
     p message_avro
     p message_avro.bytesize
     p processed_message_avro
     p processed_message_avro.bytesize
-    
+
     p message_avro.encoding
     p message_avro.valid_encoding?
 
@@ -163,7 +163,7 @@ describe RFlow::Message do
 
     message_data_avro = message.data.to_avro.force_encoding('BINARY')
     processed_message_data_avro = processed_message.data.to_avro.force_encoding('BINARY')
-    
+
     p message_data_avro.encoding
     p message_data_avro.valid_encoding?
     p message_data_avro
@@ -173,10 +173,10 @@ describe RFlow::Message do
 
     Digest::MD5.hexdigest(message_avro).should == Digest::MD5.hexdigest(processed_message_avro)
 
-    
+
     message_data_avro.should == processed_message_data_avro
     Digest::MD5.hexdigest(message_data_avro).should == Digest::MD5.hexdigest(processed_message_data_avro)
     Digest::MD5.hexdigest(message.data.raw).should == Digest::MD5.hexdigest(processed_message.data.raw)
   end
-  
+
 end

data/spec/rflow_spec.rb

@@ -1,100 +1,323 @@
 require 'spec_helper.rb'
 
+require 'open3'
 require 'rflow'
 
 describe RFlow do
-  before(:each) do
-    @fixture_directory_path = File.join(File.dirname(__FILE__), 'fixtures')
-  end
 
-  
-  describe 'logger' do
-    it "should initialize correctly" do
-      log_file_path = File.join(@temp_directory_path, 'logfile')
-      RFlow.initialize_logger log_file_path
+  before(:all) do
+    @extensions_file_name = File.join(File.dirname(__FILE__), 'fixtures', 'extensions_ints.rb')
+  end
 
-      File.exist?(log_file_path).should_not be_nil
+  context "when executing from the test script" do
 
-      RFlow.logger.error "TESTTESTTEST"
-      File.read(log_file_path).should match(/TESTTESTTEST/)
-      
-      RFlow.close_log_file
+    before(:all) do
+      load @extensions_file_name
     end
 
-    it "should reopen correctly" do
-      log_file_path = File.join(@temp_directory_path, 'logfile')
-      moved_path = log_file_path + '.old'
-      
-      RFlow.initialize_logger log_file_path
-      File.exist?(log_file_path).should be_true
-      File.exist?(moved_path).should be_false
-
-      File.rename log_file_path, moved_path
-      
-      RFlow.reopen_log_file
-
-      RFlow.logger.error "TESTTESTTEST"
-      File.read(log_file_path).should match(/TESTTESTTEST/)
-      File.read(moved_path).should_not match(/TESTTESTTEST/)
-      
-      RFlow.close_log_file
-    end
+    describe '.run' do
+      before(:each) do
+        @original_directory_path = Dir.getwd
+        @run_directory_path = File.join(@temp_directory_path, 'run')
+        @log_directory_path = File.join(@temp_directory_path, 'log')
+        Dir.mkdir @run_directory_path
+        Dir.mkdir @log_directory_path
+      end
+
+      after(:each) do
+        Dir.chdir @original_directory_path
+      end
+
+      def run_rflow_with_dsl(&block)
+        rflow_thread = Thread.new do
+          ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+          RFlow::Configuration.migrate_database
+          RFlow::Configuration::RubyDSL.configure do |c|
+            block.call(c)
+          end
+
+          RFlow::Configuration.merge_defaults!
+
+          RFlow.run nil, false
+        end
+
+        # TODO: figure out a way to get rid of this sleep, as there
+        # should be a better way to figure out when RFlow is done
+        sleep(2)
+
+        # Shut down the reactor and the thread
+        EM.run { EM.stop }
+        rflow_thread.join
+      end
+
+
+      it "should run a non-sharded workflow" do
+
+        run_rflow_with_dsl do |c|
+          c.setting('rflow.log_level', 'DEBUG')
+          c.setting('rflow.application_directory_path', @temp_directory_path)
+          c.setting('rflow.application_name', 'nonsharded_test')
+
+          c.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+          c.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => 'out'
+          c.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
+          c.component 'output_even', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_even'
+          c.component 'output_odd', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_odd'
+          c.component 'output_even_odd', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_even_odd'
+          c.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+          c.component 'output_even_odd2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_even_odd2'
+
+          c.connect 'generate_ints#out' => 'output#in'
+          c.connect 'generate_ints#out' => 'output2#in'
+          c.connect 'generate_ints#even_odd_out[even]' => 'output_even#in'
+          c.connect 'generate_ints#even_odd_out[odd]' => 'output_odd#in'
+          c.connect 'generate_ints#even_odd_out' => 'output_even_odd#in'
+          c.connect 'generate_ints2#even_odd_out' => 'output_even_odd2#in'
+        end
 
-    it "should toggle log level" do
+        RFlow.master.shards.count.should == 1
+        RFlow.master.shards.first.workers.count.should == 1
+
+        output_files = {
+          'out'           => [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
+          'out2'          => [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
+          'out_even'      => [20, 22, 24, 26, 28, 30],
+          'out_odd'       => [21, 23, 25, 27, 29],
+          'out_even_odd'  => [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
+          'out_even_odd2' => [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
+        }
+
+        output_files.each do |file_name, expected_contents|
+          File.exist?(File.join(@temp_directory_path, file_name)).should be_true
+          File.readlines(file_name).map(&:to_i).should == expected_contents
+        end
+      end
+
+
+      it "should run a sharded workflow" do
+        run_rflow_with_dsl do |c|
+          c.setting('rflow.log_level', 'DEBUG')
+          c.setting('rflow.application_directory_path', @temp_directory_path)
+          c.setting('rflow.application_name', 'sharded_test')
+
+          # Instantiate components
+          c.shard 's1', :process => 3 do |s|
+            s.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
+          end
+
+          c.shard 's2', :type => :process, :count => 2 do |s|
+            s.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+          end
+
+          c.component 'generate_ints3', 'RFlow::Components::GenerateIntegerSequence', 'start' => 100, 'finish' => 105
+
+          c.shard 's3', :process => 2 do |s|
+            s.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
+            s.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
+          end
+
+          c.component 'output3', 'RFlow::Components::FileOutput', 'output_file_path' => 'out3'
+          c.component 'output_all', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_all'
+
+          # Hook components together
+          c.connect 'generate_ints1#out' => 'output1#in'
+          c.connect 'generate_ints2#out' => 'output2#in'
+          c.connect 'generate_ints3#out' => 'output3#in'
+          c.connect 'generate_ints1#out' => 'output_all#in'
+          c.connect 'generate_ints2#out' => 'output_all#in'
+          c.connect 'generate_ints3#out' => 'output_all#in'
+        end
+
+        RFlow.master.shards.count.should == 4
+        RFlow.master.shards.map(&:count).should == [1, 3, 2, 2]
+        RFlow.master.shards.map(&:workers).map(&:count).should == [1, 3, 2, 2]
+
+        output_files = {
+          'out1'    => [0, 3, 6, 9] * 3,
+          'out2'    => (20..30).to_a * 2,
+          'out3'    => (100..105).to_a,
+          'out_all' => [0, 3, 6, 9] * 3 + (20..30).to_a * 2 + (100..105).to_a
+        }
+
+        output_files.each do |file_name, expected_contents|
+          File.exist?(File.join(@temp_directory_path, file_name)).should be_true
+          File.readlines(file_name).map(&:to_i).sort.should == expected_contents.sort
+        end
+      end
     end
   end
 
-  describe '.run' do
+  context "when executing via the rflow binary" do
     before(:each) do
+      @original_directory_path = Dir.getwd
       @run_directory_path = File.join(@temp_directory_path, 'run')
       @log_directory_path = File.join(@temp_directory_path, 'log')
       Dir.mkdir @run_directory_path
       Dir.mkdir @log_directory_path
+      Dir.chdir @temp_directory_path
     end
 
-    it "should startup and run correctly with non-trivial workflow" do
-      config_file_path = File.join(@fixture_directory_path, 'config_ints.rb')
-      extensions_path = File.join(@fixture_directory_path, 'extensions_ints.rb')
-      config_database_path = File.join(@temp_directory_path, 'config.sqlite')
+    after(:each) do
+      Dir.chdir @original_directory_path
+    end
 
-      # Load the new database with the fixtured config file
-      RFlow::Configuration::initialize_database(config_database_path, config_file_path)
-      File.exist?(config_database_path).should be_true
+    def execute_rflow(rflow_args)
+      r = {}
+      r[:stdout], r[:stderr], r[:status] = Open3.capture3("bundle exec rflow #{rflow_args}")
+      r
+    end
+
+    context "with a simple ruby DSL config file" do
+      before(:each) do
+        @config_file_name = 'input_config'
+        File.open('input_config', 'w+') do |file|
+          file.write <<-EOF
+            RFlow::Configuration::RubyDSL.configure do |c|
+              c.setting 'mysetting', 'myvalue'
+            end
+          EOF
+        end
+      end
+
+      it "should load a ruby dsl file into a sqlite DB" do
+        db_file_name = 'outdb'
+
+        r = execute_rflow("load -d #{db_file_name} -c #{@config_file_name}")
+
+        # Make sure that the process execution worked
+        r[:status].exitstatus.should == 0
+        r[:stderr].should == ''
+        r[:stdout].should match /Successfully initialized database.*#{db_file_name}/
+
+        # Make sure the config actually got loaded
+        ActiveRecord::Base.establish_connection adapter: "sqlite3", database: db_file_name
+        RFlow::Configuration::Setting.where(:name => 'mysetting').first.value.should == 'myvalue'
+      end
 
-      # Load the fixtured extensions
-      load extensions_path
+      it "should not load a database if the database file already exists" do
+        db_file_name = 'outdb'
+        File.open(db_file_name, 'w') { |file| file.write 'boom' }
 
-      # Startup RFlow in its own thread
-      rflow_thread = Thread.new do
-        RFlow.run config_database_path, false
+        r = execute_rflow("load -d #{db_file_name} -c #{@config_file_name}")
+
+        # Make sure that the process execution worked
+        r[:status].exitstatus.should == 1
+        r[:stderr].should == ''
+        r[:stdout].should match /Config database.*#{db_file_name}.*exists/
       end
 
-      # TODO: figure out a way to get rid of this sleep, as there
-      # should be a better way
-      sleep(5)
-
-      all_file_path = File.join(@temp_directory_path, 'out')
-      all2_file_path = File.join(@temp_directory_path, 'out2')
-      even_file_path = File.join(@temp_directory_path, 'out_even')
-      odd_file_path = File.join(@temp_directory_path, 'out_odd')
-      even_odd_file_path = File.join(@temp_directory_path, 'out_even_odd')
-      even_odd2_file_path = File.join(@temp_directory_path, 'out_even_odd2')
-      
-      File.exist?(all_file_path).should be_true
-      File.exist?(all2_file_path).should be_true
-      File.exist?(even_file_path).should be_true
-      File.exist?(odd_file_path).should be_true
-      File.exist?(even_odd_file_path).should be_true
-      File.exist?(even_odd2_file_path).should be_true
-
-      File.readlines(all_file_path).map(&:to_i).should == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]
-      File.readlines(all2_file_path).map(&:to_i).should == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]
-      File.readlines(even_file_path).map(&:to_i).should == [20, 22, 24, 26, 28, 30]
-      File.readlines(odd_file_path).map(&:to_i).should == [21, 23, 25, 27, 29]
-      File.readlines(even_odd_file_path).map(&:to_i).should == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]
-      File.readlines(even_odd2_file_path).map(&:to_i).should == [20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]
     end
-  end
-  
 
+    context "with a complex, sharded ruby DSL config file" do
+      before(:each) do
+        @config_file_name = 'input_config'
+        @db_file_name = 'config_db'
+        @app_name = 'sharded_bin_test'
+        File.open(@config_file_name, 'w+') do |file|
+          file.write <<-EOF
+            RFlow::Configuration::RubyDSL.configure do |c|
+              c.setting('rflow.log_level', 'INFO')
+              c.setting('rflow.application_directory_path', '#{@temp_directory_path}')
+              c.setting('rflow.application_name', '#{@app_name}')
+              # Instantiate components
+              c.shard 's1', :process => 3 do |s|
+                s.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
+              end
+              c.shard 's2', :type => :process, :count => 2 do |s|
+                s.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+              end
+              c.component 'generate_ints3', 'RFlow::Components::GenerateIntegerSequence', 'start' => 100, 'finish' => 105
+              c.shard 's3', :process => 2 do |s|
+                s.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
+                s.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
+              end
+              c.component 'output3', 'RFlow::Components::FileOutput', 'output_file_path' => 'out3'
+              c.component 'output_all', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_all'
+              # Hook components together
+              c.connect 'generate_ints1#out' => 'output1#in'
+              c.connect 'generate_ints2#out' => 'output2#in'
+              c.connect 'generate_ints3#out' => 'output3#in'
+              c.connect 'generate_ints1#out' => 'output_all#in'
+              c.connect 'generate_ints2#out' => 'output_all#in'
+              c.connect 'generate_ints3#out' => 'output_all#in'
+            end
+          EOF
+        end
+        r = execute_rflow("load -d #{@db_file_name} -c #{@config_file_name}")
+        r[:status].exitstatus.should == 0
+        r[:stderr].should == ''
+        r[:stdout].should match /Successfully initialized database.*#{@db_file_name}/
+      end
+
+      it "should not start if the components aren't loaded" do
+        r = execute_rflow("start -d #{@db_file_name} -f")
+
+        r[:status].exitstatus.should == 1
+        r[:stderr].should == ''
+        r[:stdout].should match /error/i
+      end
+
+      it "should daemonize and run in the background" do
+        r = execute_rflow("start -d #{@db_file_name} -e #{@extensions_file_name}")
+
+        r[:status].exitstatus.should == 0
+        r[:stderr].should == ''
+        r[:stdout].should_not match /error/i
+
+        sleep 1 # give the daemon a chance to finish
+
+        log_contents = File.read("log/#{@app_name}.log").chomp
+        log_lines = log_contents.split("\n")
+
+        puts '++++++++++++++++++++'
+        puts log_contents
+        puts '++++++++++++++++++++'
+
+        # Log file testing
+        log_lines.each { |line| line.should_not match /^ERROR/ }
+        log_lines.each { |line| line.should_not match /^DEBUG/ }
+
+        # Grab all the pids from the log, which seems to be the only
+        # reliable way to get them
+        log_pids = log_lines.map { |line| /\((\d+)\)/.match(line)[1].to_i }.uniq
+
+        initial_pid = r[:status].pid
+        master_pid = File.read("run/#{@app_name}.pid").chomp.to_i
+        worker_pids = log_pids - [initial_pid, master_pid]
+
+        log_pids.should include initial_pid
+        log_pids.should include master_pid
+
+        worker_pids.size.should == 8
+        worker_pids.should_not include 0
+
+        # Process checks
+        expect { Process.kill(0, initial_pid) }.to raise_error(Errno::ESRCH)
+        ([master_pid] + worker_pids).each do |pid|
+          Process.kill(0, pid).should == 1
+        end
+
+        # Output checks
+        output_files = {
+          'out1'    => [0, 3, 6, 9] * 3,
+          'out2'    => (20..30).to_a * 2,
+          'out3'    => (100..105).to_a,
+          'out_all' => [0, 3, 6, 9] * 3 + (20..30).to_a * 2 + (100..105).to_a
+        }
+
+        output_files.each do |file_name, expected_contents|
+          File.exist?(File.join(@temp_directory_path, file_name)).should be_true
+          File.readlines(file_name).map(&:to_i).sort.should == expected_contents.sort
+        end
+
+        # Terminate the master
+        Process.kill("TERM", master_pid).should == 1
+
+        # Make sure everything is dead
+        ([master_pid] + worker_pids).each do |pid|
+          expect { Process.kill(0, pid) }.to raise_error(Errno::ESRCH)
+        end
+      end
+    end
+  end
 end