rflow 0.0.5 → 1.0.0a1

data/lib/rflow/pid_file.rb

@@ -0,0 +1,84 @@
+class RFlow
+  class PIDFile
+    attr_reader :pid_file_path
+
+    def initialize(pid_file_path)
+      @pid_file_path = pid_file_path
+    end
+
+    def validate?
+      if current_process?
+        RFlow.logger.warn "Already running #{read.to_s}, not writing PID to file '#{to_s}'"
+        return nil
+      elsif running?
+        error_message = "Already running #{read.to_s}, possibly stale PID file '#{to_s}'"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      elsif exist?
+        RFlow.logger.warn "Found stale PID #{read.to_s} in PID file '#{to_s}', removing"
+        unlink
+      end
+      true
+    end
+
+    def read
+      return nil unless File.exist? pid_file_path
+      File.read(pid_file_path).to_i
+    end
+
+    def write(pid=$$)
+      return unless validate?
+
+      RFlow.logger.debug "Writing PID #{pid} file '#{to_s}'"
+      pid_fp = begin
+                 tmp_pid_file_path = File.join(File.dirname(pid_file_path), ".#{File.basename(pid_file_path)}")
+                 File.open(tmp_pid_file_path, File::RDWR|File::CREAT|File::EXCL, 0644)
+               rescue Errno::EEXIST
+                 retry
+               end
+      pid_fp.syswrite("#{pid}\n")
+      File.rename(pid_fp.path, pid_file_path)
+      pid_fp.close
+
+      pid
+    end
+
+    def exist?
+      File.exist? pid_file_path
+    end
+
+    def running?
+      return false unless exist?
+      pid = read
+      return false unless pid
+      Process.kill(0, pid)
+      pid
+    rescue Errno::ESRCH, Errno::ENOENT
+      nil
+    end
+
+    def current_process?
+      read == $$
+    end
+
+    def unlink
+      File.unlink(pid_file_path)
+    end
+
+    # unlinks a PID file at given if it contains the current PID still
+    # potentially racy without locking the directory (which is
+    # non-portable and may interact badly with other programs), but the
+    # window for hitting the race condition is small
+    def safe_unlink
+      (current_process? and unlink) rescue nil
+    end
+
+    def signal(sig)
+      Process.kill(sig, read)
+    end
+
+    def to_s
+      File.expand_path(pid_file_path)
+    end
+  end
+end

data/lib/rflow/shard.rb

@@ -0,0 +1,148 @@
+class RFlow
+
+  # An object implementation shared between two processes. The parent
+  # process will instantiate, configure, and run! a shard, at which
+  # point the parent will have access to the shard object and be able
+  # to monitor the underlying processes. The child implementation,
+  # running in a separate process, will not return from run!, but
+  # start an Eventmachine reactor, connect the components, and not
+  # return
+  class Shard
+
+    # An internal class that represents an instance of the running
+    # shard, i.e. a process
+    class Worker
+
+      attr_accessor :shard, :index, :name, :pid
+      attr_accessor :components
+      attr_accessor :worker_read, :master_write
+
+      def initialize(shard, index=1)
+        @shard = shard
+        @index = index
+        @name  = "#{shard.name}-#{index}"
+
+        # Set up the IPC pipes
+        @worker_read, @master_write = IO.pipe
+        [@worker_read, @master_write].each do |io|
+          io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+        end
+
+        @components = shard.config.components.map do |component_config|
+          Component.build(component_config)
+        end
+      end
+
+      def handle_signals
+        ['SIGTERM', 'SIGINT', 'SIGQUIT'].each do |signal|
+          Signal.trap signal do
+            Thread.new { shutdown(signal) }.join
+          end
+        end
+
+        ['SIGUSR1'].each do |signal|
+          Signal.trap signal do
+            Thread.new do
+              RFlow.logger.reopen
+            end.join
+          end
+        end
+
+        # Toggle log level on USR2
+        ['SIGUSR2'].each do |signal|
+          Signal.trap signal do
+            Thread.new do
+              RFlow.logger.toggle_log_level
+            end.join
+          end
+        end
+      end
+
+      # Launch another process to execute the shard. The parent
+      # process retains the original worker object (with pid and IPC
+      # pipe) to allow for process management
+      def launch
+        @pid = Process.fork do
+          @master_write.close
+
+          handle_signals
+
+          $0 += " #{name}"
+          Log4r::NDC.push name
+
+          RFlow.logger.info "Worker started"
+          EM.run do
+            # TODO: Monitor the master
+
+            connect_components!
+            # TODO: need to do proper node synchronization for ZMQ to
+            # remove sleep
+            sleep 1
+            run_components!
+          end
+
+          RFlow.logger.info "Shutting down worker after EM stopped"
+        end
+
+        @worker_read.close
+        self
+      end
+
+      # Send a command to each component to tell them to connect their
+      # ports via their connections
+      def connect_components!
+        RFlow.logger.debug "Connecting components"
+        components.each do |component|
+          RFlow.logger.debug "Connecting component '#{component.name}' (#{component.uuid})"
+          component.connect!
+        end
+      end
+
+      # Start each component running
+      def run_components!
+        RFlow.logger.debug "Running components"
+        components.each do |component|
+          RFlow.logger.debug "Running component '#{component.name}' (#{component.uuid})"
+          component.run!
+        end
+      end
+    end
+
+
+    attr_reader :config, :uuid, :name, :count
+    attr_accessor :workers
+
+
+    def initialize(config)
+      @config = config
+      @uuid = config.uuid
+      @name = config.name
+      @count = config.count
+
+      @workers = count.times.map do |i|
+        Worker.new(self, i+1)
+      end
+    end
+
+
+    def run!
+      RFlow.logger.debug "Running shard #{name} with #{count} workers"
+      workers.each do |worker|
+        worker.launch
+      end
+
+      RFlow.logger.debug "#{count} workers started for #{name}: #{workers.map { |w| "#{w.name} (#{w.pid})" }.join(", ")}"
+      workers
+    end
+
+
+    # TODO: Implement
+    def shutdown!
+    end
+
+
+    # TODO: Implement
+    def cleanup!
+    end
+  end
+end

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = "0.0.5"
+  VERSION = "1.0.0a1"
 end # class RFlow

data/rflow.gemspec

@@ -1,42 +1,36 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "rflow/version"
 
 Gem::Specification.new do |s|
   s.name        = "rflow"
   s.version     = RFlow::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.required_ruby_version = '~> 1.9'
+  s.required_ruby_version = ">= 1.9"
   s.authors     = ["Michael L. Artz"]
   s.email       = ["michael.artz@redjack.com"]
-  s.homepage    = ""
-  s.summary     = %q{A Ruby-based workflow framework}
-  s.description = %q{A Ruby-based workflow framework that utilizes ZeroMQ for component connections and Avro for serialization}
+  s.homepage    = "https://github.com/redjack/rflow"
+  s.license     = "Apache-2.0"
+  s.summary     = %q{A Ruby flow-based programming framework}
+  s.description = %q{A Ruby flow-based programming framework that utilizes ZeroMQ for component connections and Avro for serialization}
 
-  s.rubyforge_project = "rflow"
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files         = `git ls-files -z`.split("\x0")
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ["lib"]
 
-  s.add_dependency 'uuidtools', '~> 2.1'
-  s.add_dependency 'log4r', '~> 1.1'
-  
-  s.add_dependency 'sqlite3', '~> 1.3'
-  s.add_dependency 'activerecord', '~> 3.0'
-    
-  s.add_dependency 'avro', '>= 1.5.1'
-  s.add_dependency 'ffi', '~> 1.0'
-  s.add_dependency 'ffi-rzmq' , '~> 0.8'
+  s.add_dependency "uuidtools", "~> 2.1"
+  s.add_dependency "log4r", "~> 1.1"
+
+  s.add_dependency "sqlite3", "~> 1.3"
+  s.add_dependency "activerecord", "~> 3.2"
+
+  s.add_dependency "avro", "~> 1.7.5"
+  s.add_dependency "em-zeromq", "~> 0.4.2"
 
-  s.add_dependency 'eventmachine', '>= 1.0.0.beta3'
-  # MRI-only because of the FFI memory leak.  TODO: remove when ffi fixed
-  s.add_dependency 'em-zeromq-mri', '~> 0.2'
-  # Remove this when we break it out into its own gem
-  s.add_dependency 'eventmachine_httpserver', '~> 0.2'
-  
-  s.add_development_dependency 'rspec', '~> 2.6'
-  s.add_development_dependency 'rake', '>= 0.8.7'
-  #s.add_development_dependency 'rcov', '= 0.9.9' # Not 1.9.2 compatible
+  s.add_development_dependency "bundler", "~> 1.5"
+  s.add_development_dependency "rspec", "~> 2.6"
+  s.add_development_dependency "rake", ">= 0.8.7"
+  s.add_development_dependency "yard", "~> 0.8.7"
 end

data/schema/message.avsc

@@ -5,12 +5,12 @@
     "aliases": [],
     "fields": [
         {"name": "data_type_name", "type": "string"},
-        {"name": "provenance", 
+        {"name": "provenance",
          "type": {
-             "type": "array", 
+             "type": "array",
              "items": {
-                 "type": "record", 
-                 "name": "ProcessingEvent", 
+                 "type": "record",
+                 "name": "ProcessingEvent",
                  "namespace": "org.rflow",
                  "aliases": [],
                  "fields": [
@@ -22,10 +22,10 @@
              }
          }
         },
-        {"name": "data_serialization_type", 
+        {"name": "data_serialization_type",
          "type": {
-             "type": "enum", 
-             "name": "DataSerializationType", 
+             "type": "enum",
+             "name": "DataSerializationType",
              "symbols": ["avro", "xml"]
          }
         },
@@ -33,4 +33,4 @@
         {"name": "data", "type": "bytes"}
     ]
 }
-    
+

data/spec/fixtures/config_ints.rb

@@ -7,7 +7,7 @@ RFlow::Configuration::RubyDSL.configure do |config|
   config.setting('rflow.application_directory_path', '../tmp')
 
   config.setting('rflow.application_name', 'testapp')
-  
+
   # Add schemas to the list of available.  Not convinced this is necessary
 #  config.schema('schemaname', 'schematype', 'schemadata')
 
@@ -18,7 +18,7 @@ RFlow::Configuration::RubyDSL.configure do |config|
 #  config.component 'replicate', 'RFlow::Components::Replicate'
 #  config.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap1'
 #  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap2'
-  
+
   # Hook components together
   # config.connect 'generate_ints#out' => 'filter#in'
   # config.connect 'filter#filtered' => 'replicate#in'
@@ -50,10 +50,10 @@ RFlow::Configuration::RubyDSL.configure do |config|
   config.connect 'generate_ints#even_odd_out' => 'output_even_odd#in'
 
 
-  
+
   config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
   config.component 'output_even_odd2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd2'
-  
+
   config.connect 'generate_ints2#even_odd_out' => 'output_even_odd2#in'
 
 end

data/spec/fixtures/config_shards.rb

@@ -0,0 +1,30 @@
+RFlow::Configuration::RubyDSL.configure do |config|
+  config.setting('rflow.log_level', 'DEBUG')
+  config.setting('rflow.application_directory_path', '.')
+  config.setting('rflow.application_name', 'shardapp')
+
+  # Instantiate components
+  config.shard 's1', :process => 1 do |shard|
+    shard.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3
+  end
+
+  config.shard 's2', :type => :process, :count => 2 do |shard|
+    shard.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+  end
+
+  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'lambda {|message| true}'
+  config.component 'replicate', 'RFlow::Components::Replicate'
+
+  config.shard 's3', :process => 2 do |shard|
+    shard.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => 'out1'
+    shard.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => 'out2'
+  end
+
+  # Hook components together
+  config.connect 'generate_ints1#out' => 'filter#in'
+  config.connect 'generate_ints2#out' => 'filter#in'
+  config.connect 'filter#filtered' => 'replicate#in'
+  config.connect 'replicate#out' => 'output1#in'
+  config.connect 'replicate#out' => 'output2#in'
+
+end

data/spec/fixtures/extensions_ints.rb

@@ -25,7 +25,7 @@ RFlow::Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'a
 class RFlow::Components::GenerateIntegerSequence < RFlow::Component
   output_port :out
   output_port :even_odd_out
-  
+
   def configure!(config)
     @start = config['start'].to_i
     @finish = config['finish'].to_i
@@ -37,7 +37,7 @@ class RFlow::Components::GenerateIntegerSequence < RFlow::Component
   # Note that this uses the timer (sometimes with 0 interval) so as
   # not to block the reactor
   def run!
-    timer = EM::PeriodicTimer.new(@interval_seconds) do 
+    timer = EM::PeriodicTimer.new(@interval_seconds) do
       message = RFlow::Message.new('RFlow::Message::Data::Integer')
       message.data.data_object = @start
       out.send_message message
@@ -46,7 +46,7 @@ class RFlow::Components::GenerateIntegerSequence < RFlow::Component
       else
         even_odd_out['odd'].send_message message
       end
-      
+
       @start += @step
       timer.cancel if @start > @finish
     end
@@ -58,7 +58,7 @@ class RFlow::Components::Replicate < RFlow::Component
   input_port :in
   output_port :out
   output_port :errored
-  
+
   def process_message(input_port, input_port_key, connection, message)
     puts "Processing message in Replicate"
     out.each do |connections|
@@ -84,7 +84,7 @@ class RFlow::Components::RubyProcFilter < RFlow::Component
   def configure!(config)
     @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
   end
-  
+
   def process_message(input_port, input_port_key, connection, message)
     puts "Processing message in RubyProcFilter"
     begin
@@ -111,18 +111,18 @@ class RFlow::Components::FileOutput < RFlow::Component
   end
 
   #def run!; end
-  
+
   def process_message(input_port, input_port_key, connection, message)
     puts "About to output to a file #{output_file_path}"
     output_file.puts message.data.data_object.inspect
     output_file.flush
   end
 
-  
+
   def cleanup
     output_file.close
   end
-  
+
 end
 
 # TODO: Ensure that all the following methods work as they are