rflow 1.0.0a1 → 1.0.0a2

data/lib/rflow/shard.rb

@@ -1,148 +1,81 @@
-class RFlow
+require 'rflow/child_process'
 
+class RFlow
   # An object implementation shared between two processes. The parent
   # process will instantiate, configure, and run! a shard, at which
   # point the parent will have access to the shard object and be able
   # to monitor the underlying processes. The child implementation,
-  # running in a separate process, will not return from run!, but
-  # start an Eventmachine reactor, connect the components, and not
-  # return
+  # running in a separate process, will not return from spawn!, but
+  # start an EventMachine reactor.
   class Shard
-
-    # An internal class that represents an instance of the running
-    # shard, i.e. a process
-    class Worker
-
-      attr_accessor :shard, :index, :name, :pid
-      attr_accessor :components
-      attr_accessor :worker_read, :master_write
-
-      def initialize(shard, index=1)
+    class Worker < ChildProcess
+      def initialize(shard, index = 1)
+        super("#{shard.name}-#{index}", 'Worker')
         @shard = shard
-        @index = index
-        @name  = "#{shard.name}-#{index}"
-
-        # Set up the IPC pipes
-        @worker_read, @master_write = IO.pipe
-        [@worker_read, @master_write].each do |io|
-          io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
-        end
 
-        @components = shard.config.components.map do |component_config|
-          Component.build(component_config)
-        end
+        # build at initialize time to fail fast
+        @components = shard.config.components.map {|config| Component.build(config) }
       end
 
-      def handle_signals
-        ['SIGTERM', 'SIGINT', 'SIGQUIT'].each do |signal|
-          Signal.trap signal do
-            Thread.new { shutdown(signal) }.join
-          end
+      def run_process
+        EM.run do
+          # TODO: Monitor the master
+          configure_components!
+          connect_components!
+          # TODO: need to do proper node synchronization for ZMQ to remove sleep
+          sleep 1
+          run_components!
         end
 
-        ['SIGUSR1'].each do |signal|
-          Signal.trap signal do
-            Thread.new do
-              RFlow.logger.reopen
-            end.join
-          end
-        end
-
-        # Toggle log level on USR2
-        ['SIGUSR2'].each do |signal|
-          Signal.trap signal do
-            Thread.new do
-              RFlow.logger.toggle_log_level
-            end.join
-          end
-        end
+        RFlow.logger.info "Shutting down worker after EM stopped"
       end
 
-      # Launch another process to execute the shard. The parent
-      # process retains the original worker object (with pid and IPC
-      # pipe) to allow for process management
-      def launch
-        @pid = Process.fork do
-          @master_write.close
-
-          handle_signals
-
-          $0 += " #{name}"
-          Log4r::NDC.push name
-
-          RFlow.logger.info "Worker started"
-          EM.run do
-            # TODO: Monitor the master
-
-            connect_components!
-            # TODO: need to do proper node synchronization for ZMQ to
-            # remove sleep
-            sleep 1
-            run_components!
-          end
-
-          RFlow.logger.info "Shutting down worker after EM stopped"
+      def configure_components!
+        RFlow.logger.debug "Configuring components"
+        @components.zip(@shard.config.components.map(&:options)).each do |(component, config)|
+          RFlow.logger.debug "Configuring component '#{component.name}' (#{component.uuid})"
+          component.configure! config
         end
-
-        @worker_read.close
-        self
       end
 
-      # Send a command to each component to tell them to connect their
-      # ports via their connections
       def connect_components!
         RFlow.logger.debug "Connecting components"
-        components.each do |component|
+        @components.each do |component|
           RFlow.logger.debug "Connecting component '#{component.name}' (#{component.uuid})"
           component.connect!
         end
       end
 
-      # Start each component running
       def run_components!
         RFlow.logger.debug "Running components"
-        components.each do |component|
+        @components.each do |component|
           RFlow.logger.debug "Running component '#{component.name}' (#{component.uuid})"
           component.run!
         end
       end
-    end
 
+      def shutdown!(signal)
+        EM.stop_event_loop
+        super
+      end
+    end
 
-    attr_reader :config, :uuid, :name, :count
-    attr_accessor :workers
-
+    attr_reader :config, :name, :count, :workers
 
     def initialize(config)
       @config = config
       @uuid = config.uuid
       @name = config.name
       @count = config.count
-
-      @workers = count.times.map do |i|
-        Worker.new(self, i+1)
-      end
+      @workers = count.times.map {|i| Worker.new(self, i+1) }
     end
 
-
     def run!
       RFlow.logger.debug "Running shard #{name} with #{count} workers"
-      workers.each do |worker|
-        worker.launch
-      end
+      workers.each(&:spawn!)
 
       RFlow.logger.debug "#{count} workers started for #{name}: #{workers.map { |w| "#{w.name} (#{w.pid})" }.join(", ")}"
       workers
     end
-
-
-    # TODO: Implement
-    def shutdown!
-    end
-
-
-    # TODO: Implement
-    def cleanup!
-    end
   end
 end

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = "1.0.0a1"
-end # class RFlow
+  VERSION = "1.0.0a2"
+end

data/rflow.gemspec

@@ -1,4 +1,3 @@
-# -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "rflow/version"
@@ -30,7 +29,8 @@ Gem::Specification.new do |s|
   s.add_dependency "em-zeromq", "~> 0.4.2"
 
   s.add_development_dependency "bundler", "~> 1.5"
-  s.add_development_dependency "rspec", "~> 2.6"
+  s.add_development_dependency "rspec", "~> 2.99"
+  s.add_development_dependency "rspec-collection_matchers", "~> 0.0.4"
   s.add_development_dependency "rake", ">= 0.8.7"
   s.add_development_dependency "yard", "~> 0.8.7"
 end

data/schema/tick.avsc

@@ -0,0 +1,10 @@
+{
+  "type": "record",
+  "name": "Tick",
+  "namespace": "org.rflow.message.clock",
+  "aliases": [],
+  "fields": [
+    {"name": "name", "type": ["string", "null"]},
+    {"name": "timestamp", "type": ["long", "null"]}
+  ]
+}

data/spec/fixtures/config_ints.rb

@@ -1,61 +1,25 @@
-# Meat of the config file.  Stuff above this should probably be in
-# separate gems and/or files that are brought in at runtime.
 RFlow::Configuration::RubyDSL.configure do |config|
   # Configure the settings, which include paths for various files, log
   # levels, and component specific stuffs
-  config.setting('rflow.log_level', 'DEBUG')
+  config.setting('rflow.log_level', 'FATAL')
   config.setting('rflow.application_directory_path', '../tmp')
-
   config.setting('rflow.application_name', 'testapp')
 
-  # Add schemas to the list of available.  Not convinced this is necessary
-#  config.schema('schemaname', 'schematype', 'schemadata')
-
   # Instantiate components
-#  config.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3, 'interval_seconds' => 1
-#  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-#  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'lambda {|message| true}'
-#  config.component 'replicate', 'RFlow::Components::Replicate'
-#  config.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap1'
-#  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap2'
-
-  # Hook components together
-  # config.connect 'generate_ints#out' => 'filter#in'
-  # config.connect 'filter#filtered' => 'replicate#in'
-  # config.connect 'replicate#out[0]' => 'simple#in'
-  # config.connect 'replicate#out[one]' => 'complex#in'
-  # config.connect 'simple#out' => 'output#in'
-  # config.connect 'complex#out' => 'output#in'
-
-#  config.connect 'generate_ints1#out' => 'filter#in'
-#  config.connect 'generate_ints2#out' => 'filter#in'
-#  config.connect 'filter#filtered' => 'replicate#in'
-#  config.connect 'replicate#out[1]' => 'output1#in'
-#  config.connect 'replicate#out[2]' => 'output2#in'
-  # Some tests that should fail
-  # output should not have an 'out' ports
-#  config.connect 'output#out' => 'simple#in'
-
   config.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
   config.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out'
   config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out2'
   config.component 'output_even', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even'
   config.component 'output_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_odd'
   config.component 'output_even_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd'
+  config.component 'output_even_odd2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd2'
 
+  # Hook components together
   config.connect 'generate_ints#out' => 'output#in'
   config.connect 'generate_ints#out' => 'output2#in'
   config.connect 'generate_ints#even_odd_out[even]' => 'output_even#in'
   config.connect 'generate_ints#even_odd_out[odd]' => 'output_odd#in'
   config.connect 'generate_ints#even_odd_out' => 'output_even_odd#in'
-
-
-
-  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-  config.component 'output_even_odd2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd2'
-
   config.connect 'generate_ints2#even_odd_out' => 'output_even_odd2#in'
-
 end
-
-

data/spec/fixtures/config_shards.rb

@@ -1,5 +1,5 @@
 RFlow::Configuration::RubyDSL.configure do |config|
-  config.setting('rflow.log_level', 'DEBUG')
+  config.setting('rflow.log_level', 'FATAL')
   config.setting('rflow.application_directory_path', '.')
   config.setting('rflow.application_name', 'shardapp')
 
@@ -26,5 +26,4 @@ RFlow::Configuration::RubyDSL.configure do |config|
   config.connect 'filter#filtered' => 'replicate#in'
   config.connect 'replicate#out' => 'output1#in'
   config.connect 'replicate#out' => 'output2#in'
-
 end

data/spec/fixtures/extensions_ints.rb

@@ -1,9 +1,6 @@
-# This will/should bring in available components and their schemas
 require 'rflow/components'
 require 'rflow/message'
 
-#RFlow::Configuration.add_available_data_schema RFlow::Message::Data::AvroSchema.new('Integer', long_integer_schema)
-
 # Example of creating and registering a data extension
 module SimpleDataExtension
   # Use this to default/verify the data in data_object
@@ -15,92 +12,6 @@ module SimpleDataExtension
 end
 RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Integer', SimpleDataExtension)
 
-
-
-# Example of creating and registering a new schema
-long_integer_schema = '{"type": "long"}'
-RFlow::Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', long_integer_schema)
-
-
-class RFlow::Components::GenerateIntegerSequence < RFlow::Component
-  output_port :out
-  output_port :even_odd_out
-
-  def configure!(config)
-    @start = config['start'].to_i
-    @finish = config['finish'].to_i
-    @step = config['step'] ? config['step'].to_i : 1
-    # If interval seconds is not given, it will default to 0
-    @interval_seconds = config['interval_seconds'].to_i
-  end
-
-  # Note that this uses the timer (sometimes with 0 interval) so as
-  # not to block the reactor
-  def run!
-    timer = EM::PeriodicTimer.new(@interval_seconds) do
-      message = RFlow::Message.new('RFlow::Message::Data::Integer')
-      message.data.data_object = @start
-      out.send_message message
-      if @start % 2 == 0
-        even_odd_out['even'].send_message message
-      else
-        even_odd_out['odd'].send_message message
-      end
-
-      @start += @step
-      timer.cancel if @start > @finish
-    end
-  end
-
-end
-
-class RFlow::Components::Replicate < RFlow::Component
-  input_port :in
-  output_port :out
-  output_port :errored
-
-  def process_message(input_port, input_port_key, connection, message)
-    puts "Processing message in Replicate"
-    out.each do |connections|
-      puts "Replicating"
-      begin
-        connections.send_message message
-      rescue Exception => e
-        puts "Exception #{e.message}"
-        errored.send_message message
-      end
-    end
-  end
-end
-
-puts "Before RubyProcFilter"
-class RFlow::Components::RubyProcFilter < RFlow::Component
-  input_port :in
-  output_port :filtered
-  output_port :dropped
-  output_port :errored
-
-
-  def configure!(config)
-    @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
-  end
-
-  def process_message(input_port, input_port_key, connection, message)
-    puts "Processing message in RubyProcFilter"
-    begin
-      if @filter_proc.call(message)
-        filtered.send_message message
-      else
-        dropped.send_message message
-      end
-    rescue Exception => e
-      puts "Attempting to send message to errored #{e.message}"
-      errored.send_message message
-    end
-  end
-end
-
-puts "Before FileOutput"
 class RFlow::Components::FileOutput < RFlow::Component
   attr_accessor :output_file_path, :output_file
   input_port :in
@@ -110,23 +21,16 @@ class RFlow::Components::FileOutput < RFlow::Component
     self.output_file = File.new output_file_path, 'w+'
   end
 
-  #def run!; end
-
   def process_message(input_port, input_port_key, connection, message)
-    puts "About to output to a file #{output_file_path}"
     output_file.puts message.data.data_object.inspect
     output_file.flush
   end
 
-
   def cleanup
     output_file.close
   end
-
 end
 
-# TODO: Ensure that all the following methods work as they are
-# supposed to.  This is the interface that I'm adhering to
 class SimpleComponent < RFlow::Component
   input_port :in
   output_port :out
@@ -137,5 +41,3 @@ class SimpleComponent < RFlow::Component
   def shutdown!; end
   def cleanup!; end
 end
-
-

data/spec/rflow/component/port_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+
+class RFlow
+  class Component
+    describe Port do
+      it "should not be connected" do
+        described_class.new.should_not be_connected
+      end
+    end
+
+    describe HashPort do
+      it "should not be connected" do
+        config = double('Port Config')
+        config.stub(:name).and_return('port')
+        config.stub(:uuid).and_return('1')
+
+        described_class.new(config).should_not be_connected
+      end
+    end
+
+    describe InputPort do
+      context "#connect!" do
+        it "should be connected" do
+          connection = double('connection')
+          connection.should_receive(:connect_input!)
+
+          config = double('Port Config')
+          config.stub(:name).and_return('port')
+          config.stub(:uuid).and_return('1')
+
+          described_class.new(config).tap do |port|
+            port.add_connection(nil, connection)
+            port.should_not be_connected
+            port.connect!
+            port.should be_connected
+          end
+        end
+      end
+    end
+
+    describe OutputPort do
+      context "#connect!" do
+        it "should be connected" do
+          connection = double('connection')
+          connection.should_receive(:connect_output!)
+
+          port_config = double('Port Config')
+          port_config.stub(:name).and_return('port')
+          port_config.stub(:uuid).and_return('1')
+
+          described_class.new(port_config).tap do |port|
+            port.add_connection(nil, connection)
+            port.should_not be_connected
+            port.connect!
+            port.should be_connected
+          end
+        end
+      end
+    end
+  end
+end