rflow 1.0.0a3 → 1.0.0a4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82d81c66d6a26aa814d88893e294951247434585
-  data.tar.gz: 03edc9684d65ffb4e0729000f8948f8dbe69c16c
+  metadata.gz: 91749755100dd4bfa1eb64f8b455e42cbb9e6b1a
+  data.tar.gz: 537aa8a39d393a99e8b4f8bf674eb8d38eb87169
 SHA512:
-  metadata.gz: 7ca15c17aa77a9e15d5309c5e9dd22b533b0108ca6f8c29031899f8c824f3c4939c5c37e57bbe51db5834eefc40f0f52a840874af9660fbbe07729b8a66ea3d0
-  data.tar.gz: a8f7a76ca8eba6aec45cdf56433792938610e11332dd0523917795dd4a59edc84660d80b08a48a67c222f3a312ba6837b238a36e9db9e982b62fac2742f93417
+  metadata.gz: f107c920c6255585fe9508c50cdea49dc0c3f39f8e8cbd321cb3ec04f3997c1e733923d0fcc153dc632a134af650ec37fe55402a266f87a093ae19b431d5dcde
+  data.tar.gz: 537459d7fed23e2f025b8b06f1fa22738b39fc1235bd54376edef8d880683915769e47d5c8b94f90979bd0add24726c1749cd010f02241f3385a17d2d4ca14aa

data/bin/rflow

@@ -135,7 +135,7 @@ when 'load'
     exit 1
   end
 
-  startup_logger.warn "Config database '#{options[:config_database_path]}' not found, creating"
+  startup_logger.info "Creating config database '#{options[:config_database_path]}'"
   begin
     config = RFlow::Configuration::initialize_database(options[:config_database_path], options[:config_file_path])
   rescue Exception => e

data/example/basic_extensions.rb

@@ -30,14 +30,3 @@ class RFlow::Components::FileOutput < RFlow::Component
     end
   end
 end
-
-class SimpleComponent < RFlow::Component
-  input_port :in
-  output_port :out
-
-  def configure!(config); end
-  def run!; end
-  def process_message(input_port, input_port_key, connection, message); end
-  def shutdown!; end
-  def cleanup!; end
-end

data/example/http_extensions.rb

@@ -22,17 +22,6 @@ class RFlow::Components::FileOutput < RFlow::Component
   end
 end
 
-class SimpleComponent < RFlow::Component
-  input_port :in
-  output_port :out
-
-  def configure!(config); end
-  def run!; end
-  def process_message(input_port, input_port_key, connection, message); end
-  def shutdown!; end
-  def cleanup!; end
-end
-
 http_request_schema =<<EOS
 {
     "type": "record",

data/lib/rflow.rb

@@ -53,6 +53,7 @@ class RFlow
   end
 
   def self.start_master_node
+    RFlow.logger.info "#{configuration['rflow.application_name']} starting"
     @master = Master.new(configuration)
     master.daemonize! if @daemonize
     master.run! # blocks until EventMachine stops

data/lib/rflow/child_process.rb

@@ -78,6 +78,8 @@ class RFlow
     end
 
     def handle_signals
+      Signal.trap 'SIGCHLD', 'DEFAULT' # make sure child process can run subshells
+
       ['SIGTERM', 'SIGINT', 'SIGQUIT'].each do |signal|
         trap_signal(signal) do
           shutdown! signal

data/lib/rflow/component.rb

@@ -36,7 +36,7 @@ class RFlow
       # attempt to constantize the specification into a different
       # class. Future releases will support external (i.e. non-managed
       # components), but the current stuff only supports Ruby classes
-      def build(config)
+      def build(worker, config)
         raise NotImplementedError, "Non-managed components not yet implemented for component '#{config.name}' as '#{config.specification}' (#{config.uuid})" unless config.managed?
 
         RFlow.logger.debug "Instantiating component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
@@ -50,7 +50,7 @@ class RFlow
             component_class = config.specification.constantize
           end
 
-          component_class.new(uuid: config.uuid, name: config.name).tap do |component|
+          component_class.new(worker: worker, uuid: config.uuid, name: config.name).tap do |component|
             config.input_ports.each {|p| component.configure_input_port! p.name, uuid: p.uuid }
             config.output_ports.each {|p| component.configure_output_port! p.name, uuid: p.uuid }
 
@@ -75,17 +75,20 @@ class RFlow
     end
 
     attr_accessor :uuid, :name
-    attr_reader :ports
+    attr_reader :ports, :worker
 
     def initialize(args = {})
       @name = args[:name]
       @uuid = args[:uuid]
+      @worker = args[:worker]
       @ports = PortCollection.new
 
       self.class.defined_input_ports.each {|name, _| ports << InputPort.new(self, name: name) }
       self.class.defined_output_ports.each {|name, _| ports << OutputPort.new(self, name: name) }
     end
 
+    def shard; worker.shard if worker; end
+
     # Returns a list of connected input ports.  Each port will have
     # one or more keys associated with a particular connection.
     def input_ports; ports.by_type["RFlow::Component::InputPort"]; end

data/lib/rflow/configuration/connection.rb

@@ -104,7 +104,7 @@ class RFlow
 
     # for testing purposes
     class NullConfiguration
-      attr_accessor :name, :uuid, :options, :input_port_key, :output_port_key
+      attr_accessor :name, :uuid, :options, :input_port_key, :output_port_key, :delivery
     end
   end
 end

data/lib/rflow/configuration/migrations/20010101000004_create_connections.rb

@@ -7,6 +7,9 @@ class CreateConnections < ActiveRecord::Migration
       # To allow for multiple types of connections
       t.string :type
 
+      # round-robin or broadcast
+      t.string :delivery
+
       # Data flows from an output port to an input port
       t.string  :output_port_uuid
       t.string  :output_port_key, :default => '0'

data/lib/rflow/configuration/ruby_dsl.rb

@@ -74,12 +74,14 @@ class RFlow
       #  connect 'componentA#arrayport[2]' => 'componentB#in[1]'
       # Uses the model to assign random UUIDs
       def connect(hash)
-        hash.each do |output_string, input_string|
+        delivery = hash[:delivery] || 'round-robin'
+        hash.except(:delivery).each do |output_string, input_string|
           output_component_name, output_port_name, output_port_key = parse_connection_string(output_string)
           input_component_name, input_port_name, input_port_key = parse_connection_string(input_string)
 
           connection_specs << {
             :name => output_string + '=>' + input_string,
+            :delivery => delivery,
             :output_component_name => output_component_name,
             :output_port_name => output_port_name, :output_port_key => output_port_key,
             :output_string => output_string,
@@ -197,6 +199,7 @@ class RFlow
             connection_type = many_to_many ? RFlow::Configuration::BrokeredZMQConnection : RFlow::Configuration::ZMQConnection
 
             conn = connection_type.create!(:name => spec[:name],
+                                           :delivery => spec[:delivery],
                                            :output_port_key => spec[:output_port_key],
                                            :input_port_key => spec[:input_port_key],
                                            :output_port => output_port,
@@ -216,6 +219,18 @@ class RFlow
               conn.options['input_responsibility'] = 'connect'
             end
 
+            case spec[:delivery]
+            when 'broadcast'
+              conn.options['output_socket_type'] = 'PUB'
+              conn.options['input_socket_type'] = 'SUB'
+            when 'round-robin'
+              conn.options['output_socket_type'] = 'PUSH'
+              conn.options['input_socket_type'] = 'PULL'
+            else
+              raise RFlow::Configuration::Connection::ConnectionInvalid,
+                "Delivery type '#{spec[:delivery]}' unknown at #{spec[:config_line]}"
+            end
+
             conn.save!
             conn
           rescue Exception => e

data/lib/rflow/connections/zmq_connection.rb

@@ -6,6 +6,7 @@ end
 require 'rflow/connection'
 require 'rflow/message'
 require 'rflow/broker'
+require 'sys/filesystem'
 
 class RFlow
   module Connections
@@ -42,8 +43,13 @@ class RFlow
 
       def connect_input!
         RFlow.logger.debug "Connecting input #{uuid} with #{options.find_all {|k, v| k.to_s =~ /input/}}"
-        self.input_socket = zmq_context.socket(ZMQ.const_get(options['input_socket_type'].to_sym))
+        check_address(options['input_address'])
+
+        self.input_socket = zmq_context.socket(ZMQ.const_get(options['input_socket_type']))
         input_socket.send(options['input_responsibility'].to_sym, options['input_address'])
+        if config.delivery == 'broadcast'
+          input_socket.setsockopt(ZMQ::SUBSCRIBE, '') # request all messages
+        end
 
         input_socket.on(:message) do |*message_parts|
           begin
@@ -61,7 +67,9 @@ class RFlow
 
       def connect_output!
         RFlow.logger.debug "Connecting output #{uuid} with #{options.find_all {|k, v| k.to_s =~ /output/}}"
-        self.output_socket = zmq_context.socket(ZMQ.const_get(options['output_socket_type'].to_sym))
+        check_address(options['output_address'])
+
+        self.output_socket = zmq_context.socket(ZMQ.const_get(options['output_socket_type']))
         output_socket.send(options['output_responsibility'].to_sym, options['output_address'].to_s)
         output_socket
       end
@@ -99,6 +107,23 @@ class RFlow
 
         true
       end
+
+      def check_address(address)
+        # make sure we're not trying to create IPC sockets in an NFS share
+        # because that works poorly
+        if address.start_with?('ipc://')
+          filename = address[6..-1]
+          mount_point = Sys::Filesystem.mount_point(File.dirname(filename))
+          return unless mount_point
+          mount_type = Sys::Filesystem.mounts.find {|m| m.mount_point == mount_point }.mount_type
+          return unless mount_type
+
+          case mount_type
+          when 'vmhgfs', 'vboxsf', 'nfs' # vmware, virtualbox, nfs
+            raise ArgumentError, "Cannot safely create IPC sockets in network filesystem '#{mount_point}' of type #{mount_type}"
+          end
+        end
+      end
     end
 
     class BrokeredZMQConnection < ZMQConnection
@@ -122,11 +147,20 @@ class RFlow
         RFlow.logger.debug { "Creating a new ZeroMQ context; ZeroMQ version is #{version[:major]}.#{version[:minor]}.#{version[:patch]}" }
         @context = ZMQ::Context.new
         RFlow.logger.debug { "Connecting message broker to route from #{connection.options['output_address']} to #{connection.options['input_address']}" }
-        @back = context.socket(ZMQ::PULL)
-        back.bind(connection.options['output_address'])
-        @front = context.socket(ZMQ::PUSH)
-        front.bind(connection.options['input_address'])
-        ZMQ::Proxy.new(back, front)
+
+        @front = case connection.options['output_socket_type']
+                 when 'PUSH'; context.socket(ZMQ::PULL)
+                 when 'PUB'; context.socket(ZMQ::XSUB)
+                 else raise ArgumentError, "Unknown output socket type #{connection.options['output_socket_type']}"
+                 end
+        @back = case connection.options['input_socket_type']
+                when 'PULL'; context.socket(ZMQ::PUSH)
+                when 'SUB'; context.socket(ZMQ::XPUB)
+                else raise ArgumentError, "Unknown input socket type #{connection.options['input_socket_type']}"
+                end
+        front.bind(connection.options['output_address'])
+        back.bind(connection.options['input_address'])
+        ZMQ::Proxy.new(front, back)
         back.close
         front.close
       rescue Exception => e

data/lib/rflow/shard.rb

@@ -9,22 +9,29 @@ class RFlow
   # start an EventMachine reactor.
   class Shard
     class Worker < ChildProcess
+      attr_reader :shard, :index
+
       def initialize(shard, index = 1)
         super("#{shard.name}-#{index}", 'Worker')
         @shard = shard
+        @index = index
 
         # build at initialize time to fail fast
-        @components = shard.config.components.map {|config| Component.build(config) }
+        @components = shard.config.components.map {|config| Component.build(self, config) }
       end
 
       def run_process
         EM.run do
-          # TODO: Monitor the master
-          configure_components!
-          connect_components!
-          # TODO: need to do proper node synchronization for ZMQ to remove sleep
-          sleep 1
-          run_components!
+          begin
+            # TODO: Monitor the master
+            configure_components!
+            connect_components!
+            # TODO: need to do proper node synchronization for ZMQ to remove sleep
+            sleep 1
+            run_components!
+          rescue Exception => e
+            RFlow.logger.error "Error in worker, shutting down: #{e.class.name}: #{e.message}, because: #{e.backtrace.inspect}"
+          end
         end
 
         RFlow.logger.info "Shutting down worker after EM stopped"
@@ -32,7 +39,7 @@ class RFlow
 
       def configure_components!
         RFlow.logger.debug "Configuring components"
-        @components.zip(@shard.config.components.map(&:options)).each do |(component, config)|
+        @components.zip(shard.config.components.map(&:options)).each do |(component, config)|
           RFlow.logger.debug "Configuring component '#{component.name}' (#{component.uuid})"
           component.configure! config
         end

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = "1.0.0a3"
+  VERSION = "1.0.0a4"
 end

data/rflow.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency "uuidtools", "~> 2.1"
   s.add_dependency "log4r", "~> 1.1"
+  s.add_dependency "sys-filesystem", "~> 1.1.2"
 
   s.add_dependency "sqlite3", "~> 1.3"
   s.add_dependency "activerecord", "~> 3.2"

data/spec/fixtures/{extensions_ints.rb → extensions.rb}

@@ -30,13 +30,19 @@ class RFlow::Components::FileOutput < RFlow::Component
   end
 end
 
-class SimpleComponent < RFlow::Component
+class RFlow::Components::DateShellComponent < RFlow::Component
   input_port :in
   output_port :out
 
   def configure!(config); end
   def run!; end
-  def process_message(input_port, input_port_key, connection, message); end
+  def process_message(input_port, input_port_key, connection, message)
+    out.send_message(
+      RFlow::Message.new('RFlow::Message::Data::Raw').tap do |m|
+        m.provenance = message.provenance
+        m.data.raw = `date`
+      end)
+  end
   def shutdown!; end
   def cleanup!; end
 end

data/spec/rflow/configuration/ruby_dsl_spec.rb

@@ -39,25 +39,30 @@ class RFlow
         described_class.configure do |c|
           c.component 'first', 'First'
           c.component 'second', 'Second'
+          c.component 'third', 'Third'
           c.connect 'first#out' => 'second#in'
           c.connect 'first#out' => 'second#in[inkey]'
           c.connect 'first#out[outkey]' => 'second#in'
           c.connect 'first#out[outkey]' => 'second#in[inkey]'
+          c.connect 'second#out' => 'third#in', :delivery => 'broadcast'
         end
 
         expect(Shard).to have(1).shard
-        expect(Component).to have(2).components
-        expect(Port).to have(2).ports
-        expect(Connection).to have(4).connections
+        expect(Component).to have(3).components
+        expect(Port).to have(4).ports
+        expect(Connection).to have(5).connections
+
+        first_component = Component.find_by_name('first').tap do |c|
+          expect(c.specification).to eq('First')
+          expect(c).to have(0).input_ports
+          expect(c).to have(1).output_port
 
-        first_component = Component.where(name: 'first').first.tap do |component|
-          expect(component.specification).to eq('First')
-          expect(component).to have(0).input_ports
-          expect(component).to have(1).output_port
-          expect(component.output_ports.first.name).to eq('out')
+          out_port = c.output_ports.first
+          expect(out_port.name).to eq('out')
 
-          expect(component.output_ports.first).to have(4).connections
-          component.output_ports.first.connections.tap do |connections|
+          expect(out_port).to have(4).connections
+          out_port.connections.tap do |connections|
+            connections.each {|c| expect(c.delivery).to eq 'round-robin' }
             expect(connections[0].input_port_key).to be_nil
             expect(connections[0].output_port_key).to be_nil
             expect(connections[1].input_port_key).to eq('inkey')
@@ -69,14 +74,38 @@ class RFlow
           end
         end
 
-        Component.where(name: 'second').first.tap do |component|
-          expect(component.specification).to eq('Second')
-          expect(component).to have(1).input_port
-          expect(component.input_ports.first.name).to eq('in')
-          expect(component).to have(0).output_ports
+        second_component = Component.find_by_name('second').tap do |c|
+          expect(c.specification).to eq('Second')
+          expect(c).to have(1).input_port
+          expect(c).to have(1).output_port
+
+          in_port = c.input_ports.first
+          expect(in_port.name).to eq('in')
+
+          out_port = c.output_ports.first
+          expect(out_port.name).to eq('out')
 
-          expect(component.input_ports.first).to have(4).connections
-          expect(component.input_ports.first.connections).to eq(first_component.output_ports.first.connections)
+          expect(in_port).to have(4).connections
+          expect(in_port.connections).to eq(first_component.output_ports.first.connections)
+
+          expect(out_port).to have(1).connections
+          out_port.connections.first.tap do |connection|
+            expect(connection.delivery).to eq 'broadcast'
+            expect(connection.input_port_key).to be_nil
+            expect(connection.output_port_key).to be_nil
+          end
+        end
+
+        Component.find_by_name('third').tap do |c|
+          expect(c.specification).to eq('Third')
+          expect(c).to have(1).input_port
+          expect(c).to have(0).output_ports
+
+          in_port = c.input_ports.first
+          expect(in_port.name).to eq('in')
+
+          expect(in_port).to have(1).connections
+          expect(in_port.connections).to eq(second_component.output_ports.first.connections)
         end
       end
 
@@ -316,6 +345,76 @@ class RFlow
         end
       end
 
+      it "should generate PUB-SUB ipc ZeroMQ connections for one-to-many broadcast connections" do
+        described_class.configure do |c|
+
+          c.shard "s1", :process => 1 do |s|
+            s.component 'first', 'First', :opt1 => 'opt1'
+          end
+
+          c.shard "s2", :process => 3 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          end
+
+          c.connect 'first#out' => 'second#in', :delivery => 'broadcast'
+        end
+
+        expect(Shard).to have(2).shards
+        expect(Component).to have(2).components
+        expect(Port).to have(2).ports
+        expect(Connection).to have(1).connections
+
+        Connection.first.tap do |conn|
+          expect(conn.type).to eq('RFlow::Configuration::ZMQConnection')
+          expect(conn.name).to eq('first#out=>second#in')
+          expect(conn.output_port_key).to be_nil
+          expect(conn.input_port_key).to be_nil
+          conn.options.tap do |opts|
+            expect(opts['output_socket_type']).to eq('PUB')
+            expect(opts['output_address']).to eq("ipc://rflow.#{conn.uuid}")
+            expect(opts['output_responsibility']).to eq('bind')
+            expect(opts['input_socket_type']).to eq('SUB')
+            expect(opts['input_address']).to eq("ipc://rflow.#{conn.uuid}")
+            expect(opts['input_responsibility']).to eq('connect')
+          end
+        end
+      end
+
+      it "should generate PUB-SUB brokered ZeroMQ connections for many-to-many broadcast connections" do
+        described_class.configure do |c|
+
+          c.shard "s1", :process => 3 do |s|
+            s.component 'first', 'First', :opt1 => 'opt1'
+          end
+
+          c.shard "s2", :process => 3 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          end
+
+          c.connect 'first#out' => 'second#in', :delivery => 'broadcast'
+        end
+
+        expect(Shard).to have(2).shards
+        expect(Component).to have(2).components
+        expect(Port).to have(2).ports
+        expect(Connection).to have(1).connections
+
+        Connection.first.tap do |conn|
+          expect(conn.type).to eq('RFlow::Configuration::BrokeredZMQConnection')
+          expect(conn.name).to eq('first#out=>second#in')
+          expect(conn.output_port_key).to be_nil
+          expect(conn.input_port_key).to be_nil
+          conn.options.tap do |opts|
+            expect(opts['output_socket_type']).to eq('PUB')
+            expect(opts['output_address']).to eq("ipc://rflow.#{conn.uuid}.in")
+            expect(opts['output_responsibility']).to eq('connect')
+            expect(opts['input_socket_type']).to eq('SUB')
+            expect(opts['input_address']).to eq("ipc://rflow.#{conn.uuid}.out")
+            expect(opts['input_responsibility']).to eq('connect')
+          end
+        end
+      end
+
       it "should not allow two components with the same name" do
         expect {
           described_class.configure do |c|

data/spec/rflow_spec.rb

@@ -3,8 +3,84 @@ require 'open3'
 require 'rflow'
 
 describe RFlow do
+  def write_config_file(content)
+    File.open(config_file_name, 'w+') {|file| file.write content }
+  end
+
+  def execute_rflow(args)
+    stdout, stderr, status = Open3.capture3("bundle exec rflow #{args}")
+    {:stdout => stdout, :stderr => stderr, :status => status}
+  end
+
+  def load_database
+    execute_rflow("load -d #{db_file_name} -c #{config_file_name}").tap do |result|
+      expect(result[:status].exitstatus).to eq(0)
+      expect(result[:stderr]).to eq('')
+      expect(result[:stdout]).to match /Successfully initialized database.*#{db_file_name}/
+    end
+  end
+
+  def start_rflow
+    execute_rflow("start -d #{db_file_name} -e #{@extensions_file_name}").tap do |result|
+      expect(result[:status].exitstatus).to eq(0)
+      expect(result[:stderr]).to eq('')
+      expect(result[:stdout]).not_to match /error/i
+    end
+  end
+
+  def get_log_pids(logfile)
+    log_contents = File.read(logfile).chomp
+    log_lines = log_contents.split("\n")
+
+    log_lines.each {|line| expect(line).not_to match /^ERROR/ }
+    log_lines.each {|line| expect(line).not_to match /^DEBUG/ }
+
+    # Grab all the pids from the log, which seems to be the only
+    # reliable way to get them
+    log_lines.map {|line| /\((\d+)\)/.match(line)[1].to_i }.uniq
+  end
+
+  def run_and_shutdown(app_name, expected_worker_count)
+    r = start_rflow
+    sleep 2 # give the daemon a chance to finish
+
+    log_pids = get_log_pids("log/#{app_name}.log")
+
+    initial_pid = r[:status].pid
+    master_pid = File.read("run/#{app_name}.pid").chomp.to_i
+    worker_pids = log_pids - [initial_pid, master_pid]
+
+    expect(log_pids).to include initial_pid
+    expect(log_pids).to include master_pid
+
+    expect(worker_pids).to have(expected_worker_count).pids
+    expect(worker_pids).not_to include 0
+
+    expect { Process.kill(0, initial_pid) }.to raise_error(Errno::ESRCH)
+    ([master_pid] + worker_pids).each do |pid|
+      expect(Process.kill(0, pid)).to eq(1)
+    end
+
+    yield # verify output
+
+    # Terminate the master
+    expect(Process.kill("TERM", master_pid)).to eq(1)
+
+    # Make sure everything is dead after a second
+    sleep 2
+    ([master_pid] + worker_pids).each do |pid|
+      expect { Process.kill(0, pid) }.to raise_error(Errno::ESRCH)
+    end
+  rescue Exception => e
+    Process.kill("TERM", master_pid) if master_pid
+    raise
+  end
+
+  let(:config_file_name) { 'input_config' }
+  let(:db_file_name) { 'outdb' }
+
   before(:all) do
-    @extensions_file_name = File.join(File.dirname(__FILE__), 'fixtures', 'extensions_ints.rb')
+    @extensions_file_name = File.join(File.dirname(__FILE__), 'fixtures', 'extensions.rb')
   end
 
   before(:each) do
@@ -43,7 +119,7 @@ describe RFlow do
 
       it "should run a non-sharded workflow" do
         run_rflow_with_dsl do |c|
-          c.setting 'rflow.log_level', 'FATAL'
+          c.setting 'rflow.log_level', 'ERROR'
           c.setting 'rflow.application_directory_path', @temp_directory_path
           c.setting 'rflow.application_name', 'nonsharded_test'
 
@@ -84,7 +160,7 @@ describe RFlow do
 
       it "should run a sharded workflow" do
         run_rflow_with_dsl do |c|
-          c.setting 'rflow.log_level', 'FATAL'
+          c.setting 'rflow.log_level', 'ERROR'
           c.setting 'rflow.application_directory_path', @temp_directory_path
           c.setting 'rflow.application_name', 'sharded_test'
 
@@ -130,38 +206,62 @@ describe RFlow do
           expect(File.readlines(file_name).map(&:to_i).sort).to eq(expected_contents.sort)
         end
       end
+
+      it "should deliver broadcast messages to every copy of a shard" do
+        run_rflow_with_dsl do |c|
+          c.setting 'rflow.log_level', 'FATAL'
+          c.setting 'rflow.application_directory_path', @temp_directory_path
+          c.setting 'rflow.application_name', 'sharded_broadcast_test'
+
+          c.shard 's1', :process => 1 do |s|
+            s.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
+          end
+
+          c.shard 's2', :process => 2 do |s|
+            s.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 1, 'finish' => 11, 'step' => 3
+          end
+
+          c.shard 's3', :type => :process, :count => 3 do |s|
+            s.component 'broadcast_output', 'RFlow::Components::FileOutput', 'output_file_path' => 'broadcast'
+            s.component 'roundrobin_output', 'RFlow::Components::FileOutput', 'output_file_path' => 'round-robin'
+          end
+
+          c.connect 'generate_ints1#out' => 'broadcast_output#in', :delivery => 'broadcast'
+          c.connect 'generate_ints2#out' => 'broadcast_output#in', :delivery => 'broadcast'
+          c.connect 'generate_ints1#out' => 'roundrobin_output#in'
+          c.connect 'generate_ints2#out' => 'roundrobin_output#in'
+        end
+
+        output_files = {
+          'broadcast'   => ([0, 3, 6, 9] * 3) + ([1, 4, 7, 10] * 6),
+          'round-robin' => [0, 3, 6, 9] + ([1, 4, 7, 10] * 2)
+        }
+
+        expect(RFlow.master).to have(3).shards
+        expect(RFlow.master.shards.map(&:count)).to eq([1, 2, 3])
+        expect(RFlow.master.shards.map(&:workers).map(&:count)).to eq([1, 2, 3])
+
+        output_files.each do |file_name, expected_contents|
+          expect(File.exist?(File.join(@temp_directory_path, file_name))).to be true
+          expect(File.readlines(file_name).map(&:to_i).sort).to eq(expected_contents.sort)
+        end
+      end
     end
   end
 
   context "when executing via the rflow binary" do
-    def execute_rflow(args)
-      stdout, stderr, status = Open3.capture3("bundle exec rflow #{args}")
-      {:stdout => stdout, :stderr => stderr, :status => status}
-    end
-
     context "with a simple ruby DSL config file" do
-      let(:config_file_name) { 'input_config' }
-      let(:db_file_name) { 'outdb' }
-
       before(:each) do
-        File.open(config_file_name, 'w+') do |file|
-          file.write <<-EOF
-            RFlow::Configuration::RubyDSL.configure do |c|
-              c.setting 'mysetting', 'myvalue'
-            end
-          EOF
-        end
+        write_config_file <<-EOF
+          RFlow::Configuration::RubyDSL.configure do |c|
+            c.setting 'mysetting', 'myvalue'
+          end
+        EOF
       end
 
       it "should load a ruby dsl file into a sqlite DB" do
-        r = execute_rflow("load -d #{db_file_name} -c #{config_file_name}")
-
-        # Make sure that the process execution worked
-        expect(r[:status].exitstatus).to eq(0)
-        expect(r[:stderr]).to eq('')
-        expect(r[:stdout]).to match /Successfully initialized database.*#{db_file_name}/
+        load_database
 
-        # Make sure the config actually got loaded
         ActiveRecord::Base.establish_connection adapter: "sqlite3", database: db_file_name
         expect(RFlow::Configuration::Setting.where(:name => 'mysetting').first.value).to eq('myvalue')
       end
@@ -178,46 +278,70 @@ describe RFlow do
       end
     end
 
+    context "with a component that runs subshells" do
+      let(:app_name) { 'sharded_subshell_test' }
+
+      before(:each) do
+        write_config_file <<-EOF
+          RFlow::Configuration::RubyDSL.configure do |c|
+            c.setting('rflow.log_level', 'INFO')
+            c.setting('rflow.application_directory_path', '#{@temp_directory_path}')
+            c.setting('rflow.application_name', '#{app_name}')
+
+            c.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
+            c.component 'subshell_date', 'RFlow::Components::DateShellComponent'
+            c.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
+
+            c.connect 'generate_ints#out' => 'subshell_date#in'
+            c.connect 'subshell_date#out' => 'output#in'
+          end
+        EOF
+
+        load_database
+      end
+
+      it "should run successfully daemonize and run in the background" do
+        run_and_shutdown app_name, 1 do # 1 default worker
+          expect(File.exist?(File.join(@temp_directory_path, 'out1'))).to be true
+          File.readlines('out1').each {|line| expect(line).to match /\w+ \w+ \d+ \d+:\d+:\d+ \w+ \d+/ }
+        end
+      end
+    end
+
     context "with a complex, sharded ruby DSL config file" do
-      let(:config_file_name) { 'input_config' }
-      let(:db_file_name) { 'config_db' }
       let(:app_name) { 'sharded_bin_test' }
 
       before(:each) do
-        File.open(config_file_name, 'w+') do |file|
-          file.write <<-EOF
-            RFlow::Configuration::RubyDSL.configure do |c|
-              c.setting('rflow.log_level', 'INFO')
-              c.setting('rflow.application_directory_path', '#{@temp_directory_path}')
-              c.setting('rflow.application_name', '#{app_name}')
-
-              c.shard 's1', :process => 3 do |s|
-                s.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
-              end
-              c.shard 's2', :type => :process, :count => 2 do |s|
-                s.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-              end
-              c.component 'generate_ints3', 'RFlow::Components::GenerateIntegerSequence', 'start' => 100, 'finish' => 105
-              c.shard 's3', :process => 2 do |s|
-                s.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
-                s.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
-              end
-              c.component 'output3', 'RFlow::Components::FileOutput', 'output_file_path' => 'out3'
-              c.component 'output_all', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_all'
-
-              c.connect 'generate_ints1#out' => 'output1#in'
-              c.connect 'generate_ints2#out' => 'output2#in'
-              c.connect 'generate_ints3#out' => 'output3#in'
-              c.connect 'generate_ints1#out' => 'output_all#in'
-              c.connect 'generate_ints2#out' => 'output_all#in'
-              c.connect 'generate_ints3#out' => 'output_all#in'
+        write_config_file <<-EOF
+          RFlow::Configuration::RubyDSL.configure do |c|
+            c.setting('rflow.log_level', 'INFO')
+            c.setting('rflow.application_directory_path', '#{@temp_directory_path}')
+            c.setting('rflow.application_name', '#{app_name}')
+
+            c.shard 's1', :process => 3 do |s|
+              s.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
             end
-          EOF
-        end
-        r = execute_rflow("load -d #{db_file_name} -c #{config_file_name}")
-        expect(r[:status].exitstatus).to eq(0)
-        expect(r[:stderr]).to eq('')
-        expect(r[:stdout]).to match /Successfully initialized database.*#{db_file_name}/
+            c.shard 's2', :type => :process, :count => 2 do |s|
+              s.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+            end
+            c.component 'generate_ints3', 'RFlow::Components::GenerateIntegerSequence', 'start' => 100, 'finish' => 105
+            c.shard 's3', :process => 2 do |s|
+              s.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
+              s.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
+            end
+            c.component 'output3', 'RFlow::Components::FileOutput', 'output_file_path' => 'out3'
+            c.component 'output_all', 'RFlow::Components::FileOutput', 'output_file_path' => 'out_all'
+
+            c.connect 'generate_ints1#out' => 'output1#in'
+            c.connect 'generate_ints2#out' => 'output2#in'
+            c.connect 'generate_ints3#out' => 'output3#in'
+            c.connect 'generate_ints1#out' => 'output_all#in'
+            c.connect 'generate_ints2#out' => 'output_all#in'
+            c.connect 'generate_ints3#out' => 'output_all#in'
+          end
+        EOF
+
+        load_database
       end
 
       it "should not start if the components aren't loaded" do
@@ -229,63 +353,18 @@ describe RFlow do
       end
 
       it "should daemonize and run in the background" do
-        begin
-          r = execute_rflow("start -d #{db_file_name} -e #{@extensions_file_name}")
-
-          expect(r[:status].exitstatus).to eq(0)
-          expect(r[:stderr]).to eq('')
-          expect(r[:stdout]).not_to match /error/i
-
-          sleep 2 # give the daemon a chance to finish
-
-          log_contents = File.read("log/#{app_name}.log").chomp
-          log_lines = log_contents.split("\n")
-
-          log_lines.each {|line| expect(line).not_to match /^ERROR/ }
-          log_lines.each {|line| expect(line).not_to match /^DEBUG/ }
-
-          # Grab all the pids from the log, which seems to be the only
-          # reliable way to get them
-          log_pids = log_lines.map {|line| /\((\d+)\)/.match(line)[1].to_i }.uniq
-
-          initial_pid = r[:status].pid
-          master_pid = File.read("run/#{app_name}.pid").chomp.to_i
-          worker_pids = log_pids - [initial_pid, master_pid]
-
-          expect(log_pids).to include initial_pid
-          expect(log_pids).to include master_pid
-
-          expect(worker_pids).to have(10).pids # 1+3+2+2 workers, 2 brokers
-          expect(worker_pids).not_to include 0
-
-          expect { Process.kill(0, initial_pid) }.to raise_error(Errno::ESRCH)
-          ([master_pid] + worker_pids).each do |pid|
-            expect(Process.kill(0, pid)).to eq(1)
-          end
-
-          output_files = {
-            'out1'    => [0, 3, 6, 9] * 3,
-            'out2'    => (20..30).to_a * 2,
-            'out3'    => (100..105).to_a,
-            'out_all' => [0, 3, 6, 9] * 3 + (20..30).to_a * 2 + (100..105).to_a
-          }
+        output_files = {
+          'out1'    => [0, 3, 6, 9] * 3,
+          'out2'    => (20..30).to_a * 2,
+          'out3'    => (100..105).to_a,
+          'out_all' => [0, 3, 6, 9] * 3 + (20..30).to_a * 2 + (100..105).to_a
+        }
 
+        run_and_shutdown app_name, 10 do # 1+3+2+2 workers, 2 brokers
           output_files.each do |file_name, expected_contents|
             expect(File.exist?(File.join(@temp_directory_path, file_name))).to be true
             expect(File.readlines(file_name).map(&:to_i).sort).to eq(expected_contents.sort)
           end
-
-          # Terminate the master
-          expect(Process.kill("TERM", master_pid)).to eq(1)
-
-          # Make sure everything is dead after a second
-          sleep 2
-          ([master_pid] + worker_pids).each do |pid|
-            expect { Process.kill(0, pid) }.to raise_error(Errno::ESRCH)
-          end
-        rescue Exception => e
-          Process.kill("TERM", master_pid) if master_pid
-          raise
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rflow
 version: !ruby/object:Gem::Version
-  version: 1.0.0a3
+  version: 1.0.0a4
 platform: ruby
 authors:
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-16 00:00:00.000000000 Z
+date: 2014-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: sys-filesystem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -230,9 +244,7 @@ files:
 - schema/message.avsc
 - schema/raw.avsc
 - schema/tick.avsc
-- spec/fixtures/config_ints.rb
-- spec/fixtures/config_shards.rb
-- spec/fixtures/extensions_ints.rb
+- spec/fixtures/extensions.rb
 - spec/rflow/component/port_spec.rb
 - spec/rflow/components/clock_spec.rb
 - spec/rflow/configuration/ruby_dsl_spec.rb
@@ -270,9 +282,7 @@ signing_key:
 specification_version: 4
 summary: A Ruby flow-based programming framework
 test_files:
-- spec/fixtures/config_ints.rb
-- spec/fixtures/config_shards.rb
-- spec/fixtures/extensions_ints.rb
+- spec/fixtures/extensions.rb
 - spec/rflow/component/port_spec.rb
 - spec/rflow/components/clock_spec.rb
 - spec/rflow/configuration/ruby_dsl_spec.rb

data/spec/fixtures/config_ints.rb

@@ -1,25 +0,0 @@
-RFlow::Configuration::RubyDSL.configure do |config|
-  # Configure the settings, which include paths for various files, log
-  # levels, and component specific stuffs
-  config.setting('rflow.log_level', 'FATAL')
-  config.setting('rflow.application_directory_path', '../tmp')
-  config.setting('rflow.application_name', 'testapp')
-
-  # Instantiate components
-  config.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-  config.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out'
-  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out2'
-  config.component 'output_even', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even'
-  config.component 'output_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_odd'
-  config.component 'output_even_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd'
-  config.component 'output_even_odd2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd2'
-
-  # Hook components together
-  config.connect 'generate_ints#out' => 'output#in'
-  config.connect 'generate_ints#out' => 'output2#in'
-  config.connect 'generate_ints#even_odd_out[even]' => 'output_even#in'
-  config.connect 'generate_ints#even_odd_out[odd]' => 'output_odd#in'
-  config.connect 'generate_ints#even_odd_out' => 'output_even_odd#in'
-  config.connect 'generate_ints2#even_odd_out' => 'output_even_odd2#in'
-end

data/spec/fixtures/config_shards.rb

@@ -1,29 +0,0 @@
-RFlow::Configuration::RubyDSL.configure do |config|
-  config.setting('rflow.log_level', 'FATAL')
-  config.setting('rflow.application_directory_path', '.')
-  config.setting('rflow.application_name', 'shardapp')
-
-  # Instantiate components
-  config.shard 's1', :process => 1 do |shard|
-    shard.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
-  end
-
-  config.shard 's2', :type => :process, :count => 2 do |shard|
-    shard.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-  end
-
-  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'lambda {|message| true}'
-  config.component 'replicate', 'RFlow::Components::Replicate'
-
-  config.shard 's3', :process => 2 do |shard|
-    shard.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
-    shard.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
-  end
-
-  # Hook components together
-  config.connect 'generate_ints1#out' => 'filter#in'
-  config.connect 'generate_ints2#out' => 'filter#in'
-  config.connect 'filter#filtered' => 'replicate#in'
-  config.connect 'replicate#out' => 'output1#in'
-  config.connect 'replicate#out' => 'output2#in'
-end