rflow 1.0.0a1 → 1.0.0a2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 57c6f0b7c61b30886bbf0f4b2f65821aa5b1b0f9
-  data.tar.gz: 62f58d281509732effeca0c1a041df2668497b80
+  metadata.gz: 01773e5fd8c83119dd4c8f699099848bf3584738
+  data.tar.gz: b57b8e3c110e826c8d5db30e1c43e416c56bc627
 SHA512:
-  metadata.gz: 8d74949a15024641aef4123ca703d2f2ccf6fb5f97dca9829a282ca53bd6d36c347c844b189255955c7fa058bf903853d0c3acf13fda4dc2e2b3f40e49129310
-  data.tar.gz: f6233d9cc128220c886b6ed4970b544040cace77af6701b6f7429da304ad7de00b5536455469e21988f37d2fa1faaedf3f4324f08eee343669b03c6bdaece735
+  metadata.gz: 4c51ebe3e627932117535eeb8cfb0cb213e34fccf9938b78a27095aaf9da2792a2f1e16ddc66bb49a4556adbbe7ca37ed0d96a048319c18581546b91fcbf7d29
+  data.tar.gz: ad1162c1f0a2902300073cdccc0ccab3a67ace9fd99cfc47c5bd8449c8d551e94e089a4a73e15600910f08a87de7e34d2d8751ba204a891a62f0a35103618fee

data/.gitignore

@@ -4,3 +4,5 @@ Gemfile.lock
 pkg/*
 \#*
 .\#*
+spec/tmp/
+*.swp

data/.rspec

@@ -0,0 +1 @@
+--color -fd

data/Gemfile

@@ -1,6 +1,5 @@
 source "https://rubygems.org"
 
-# Specify your gem's dependencies in rflow.gemspec
 gemspec
 
 group :development do

data/NOTES

@@ -1,16 +1,3 @@
-RFlow starts
-read in DB
-create new shards
-  - Create a set of workers with the shard configuration
-    - each worker creates a set of components
-
-  - create components
-
-
-
-
-
-
 RFlow Manager
 
   Components

data/README.md

@@ -28,8 +28,13 @@ processes and/or threads.
 
 Some of the long-term goals of RFlow are to allow for components and
 portions of the workflow to be defined in any language that supports
-Avro and ZeroMQ, which a numerous.
+Avro and ZeroMQ, which are numerous.
 
+## Developer Notes
+
+You will need ZeroMQ preinstalled. Currently, EventMachine only supports
+v3.2.4, not v4.x, so install that version. Older versions like 2.2 will not
+work.
 
 ## Definitions
 

data/bin/rflow

@@ -97,13 +97,11 @@ if options[:config_file_path] && command != 'load'
   exit 1
 end
 
-
 unless options[:config_database_path]
   startup_logger.warn "Config database not specified, using default 'config.sqlite'"
   options[:config_database_path] = File.expand_path(File.join(Dir.getwd, 'config.sqlite'))
 end
 
-
 # Set the standard logger to the startup one in the case that we need
 # to call into RFlow to check on or setup things, like the config
 # database.  We want those log messages to go to the startup log when
@@ -111,7 +109,6 @@ end
 # the config database
 RFlow.logger = startup_logger
 
-
 case command
 when 'load'
   # Load the database with the config file, if it exists.  Will
@@ -146,7 +143,6 @@ when 'load'
   exit 0
 end
 
-
 # Load the database config and start setting up environment
 begin
   config = RFlow::Configuration.new(options[:config_database_path])
@@ -159,7 +155,6 @@ Dir.chdir(File.dirname(options[:config_database_path]))
 Dir.chdir(config['rflow.application_directory_path'])
 pid_file = RFlow::PIDFile.new(config['rflow.pid_file_path'])
 
-
 case command
 when 'stop'
   if pid_file.running?
@@ -187,7 +182,6 @@ when 'start'
   end
 end
 
-
 # We should have eliminated all commands but 'start' at this point
 
 # require all the gem extensions
@@ -196,7 +190,6 @@ options[:gems].each do |extension_gem|
   require extension_gem
 end
 
-
 # load all the file extensions
 options[:extensions_file_paths].each do |extensions_file_path|
   startup_logger.info "Loading #{extensions_file_path}"
@@ -207,12 +200,12 @@ options[:extensions_file_paths].each do |extensions_file_path|
   load extensions_file_path
 end
 
-
 # Start the flow
 begin
-  RFlow.run options[:config_database_path], options[:daemonize]
+  RFlow.run! options[:config_database_path], options[:daemonize]
 rescue Exception => e
   startup_logger.fatal "Error running rflow: #{e.class}: #{e.message}"
+  exit(1)
 end
 
 __END__

data/example/basic_config.rb

@@ -1,49 +1,17 @@
-# Meat of the config file.  Stuff above this should probably be in
-# separate gems and/or files that are brought in at runtime.
 RFlow::Configuration::RubyDSL.configure do |config|
   # Configure the settings, which include paths for various files, log
   # levels, and component specific stuffs
   config.setting('rflow.log_level', 'DEBUG')
   config.setting('rflow.application_directory_path', '.')
 
-  # Add schemas to the list of available.  Not convinced this is necessary
-#  config.schema('schemaname', 'schematype', 'schemadata')
-
   # Instantiate components
-#  config.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3, 'interval_seconds' => 1
-#  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
-#  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'lambda {|message| true}'
-#  config.component 'replicate', 'RFlow::Components::Replicate'
-#  config.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap1'
-#  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap2'
-
-  # Hook components together
-  # config.connect 'generate_ints#out' => 'filter#in'
-  # config.connect 'filter#filtered' => 'replicate#in'
-  # config.connect 'replicate#out[0]' => 'simple#in'
-  # config.connect 'replicate#out[one]' => 'complex#in'
-  # config.connect 'simple#out' => 'output#in'
-  # config.connect 'complex#out' => 'output#in'
-
-#  config.connect 'generate_ints1#out' => 'filter#in'
-#  config.connect 'generate_ints2#out' => 'filter#in'
-#  config.connect 'filter#filtered' => 'replicate#in'
-#  config.connect 'replicate#out[1]' => 'output1#in'
-#  config.connect 'replicate#out[2]' => 'output2#in'
-  # Some tests that should fail
-  # output should not have an 'out' ports
-#  config.connect 'output#out' => 'simple#in'
-
   config.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
   config.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap'
   config.component 'output_even', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap_even'
   config.component 'output_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap_odd'
 
+  # Hook components together
   config.connect 'generate_ints#even_odd_out' => 'output#in'
   config.connect 'generate_ints#even_odd_out[even]' => 'output_even#in'
   config.connect 'generate_ints#even_odd_out[odd]' => 'output_odd#in'
-
-
 end
-
-

data/example/basic_extensions.rb

@@ -2,8 +2,6 @@
 require 'rflow/components'
 require 'rflow/message'
 
-#RFlow::Configuration.add_available_data_schema RFlow::Message::Data::AvroSchema.new('Integer', long_integer_schema)
-
 # Example of creating and registering a data extension
 module SimpleDataExtension
   # Use this to default/verify the data in data_object
@@ -15,93 +13,6 @@ module SimpleDataExtension
 end
 RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Integer', SimpleDataExtension)
 
-
-
-# Example of creating and registering a new schema
-long_integer_schema = '{"type": "long"}'
-RFlow::Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', long_integer_schema)
-
-
-class RFlow::Components::GenerateIntegerSequence < RFlow::Component
-  output_port :out
-  output_port :even_odd_out
-
-  def configure!(config)
-    @start = config['start'].to_i
-    @finish = config['finish'].to_i
-    @step = config['step'] ? config['step'].to_i : 1
-    # If interval seconds is not given, it will default to 0
-    @interval_seconds = config['interval_seconds'].to_i
-  end
-
-  # Note that this uses the timer (sometimes with 0 interval) so as
-  # not to block the reactor
-  def run!
-    timer = EM::PeriodicTimer.new(@interval_seconds) do
-      message = RFlow::Message.new('RFlow::Message::Data::Integer')
-      message.data.data_object = @start
-      out.send_message message
-      if @start % 2 == 0
-        even_odd_out['even'].send_message message
-      else
-        even_odd_out['odd'].send_message message
-      end
-      even_odd_out.send_message message
-
-      @start += @step
-      timer.cancel if @start > @finish
-    end
-  end
-
-end
-
-class RFlow::Components::Replicate < RFlow::Component
-  input_port :in
-  output_port :out
-  output_port :errored
-
-  def process_message(input_port, input_port_key, connection, message)
-    puts "Processing message in Replicate"
-    out.each do |connections|
-      puts "Replicating"
-      begin
-        connections.send_message message
-      rescue Exception => e
-        puts "Exception #{e.message}"
-        errored.send_message message
-      end
-    end
-  end
-end
-
-puts "Before RubyProcFilter"
-class RFlow::Components::RubyProcFilter < RFlow::Component
-  input_port :in
-  output_port :filtered
-  output_port :dropped
-  output_port :errored
-
-
-  def configure!(config)
-    @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
-  end
-
-  def process_message(input_port, input_port_key, connection, message)
-    puts "Processing message in RubyProcFilter"
-    begin
-      if @filter_proc.call(message)
-        filtered.send_message message
-      else
-        dropped.send_message message
-      end
-    rescue Exception => e
-      puts "Attempting to send message to errored #{e.message}"
-      errored.send_message message
-    end
-  end
-end
-
-puts "Before FileOutput"
 class RFlow::Components::FileOutput < RFlow::Component
   attr_accessor :output_file_path, :output_file
   input_port :in
@@ -111,23 +22,16 @@ class RFlow::Components::FileOutput < RFlow::Component
     self.output_file = File.new output_file_path, 'w+'
   end
 
-  #def run!; end
-
   def process_message(input_port, input_port_key, connection, message)
-    puts "About to output to a file #{output_file_path}"
     output_file.puts message.data.data_object.inspect
     output_file.flush
   end
 
-
   def cleanup
     output_file.close
   end
-
 end
 
-# TODO: Ensure that all the following methods work as they are
-# supposed to.  This is the interface that I'm adhering to
 class SimpleComponent < RFlow::Component
   input_port :in
   output_port :out
@@ -138,5 +42,3 @@ class SimpleComponent < RFlow::Component
   def shutdown!; end
   def cleanup!; end
 end
-
-

data/example/http_config.rb

@@ -11,11 +11,10 @@ RFlow::Configuration::RubyDSL.configure do |config|
   config.component 'file_output', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/http_crap'
   config.component 'http_responder', 'HTTPResponder', 'response_code' => 200, 'content' => 'Hi, this teh awesome'
 
-#  config.connect 'http_server#request_port' => 'filter#in'
-#  config.connect 'filter#filtered' => 'replicate#in'
+  #config.connect 'http_server#request_port' => 'filter#in'
+  #config.connect 'filter#filtered' => 'replicate#in'
   config.connect 'http_server#request_port' => 'replicate#in'
   config.connect 'replicate#out[1]' => 'file_output#in'
   config.connect 'replicate#out[2]' => 'http_responder#request'
   config.connect 'http_responder#response' => 'http_server#response_port'
 end
-

data/example/http_extensions.rb

@@ -1,52 +1,8 @@
 # This will/should bring in available components and their schemas
 require 'rflow/components'
 require 'rflow/message'
-
-
-class RFlow::Components::Replicate < RFlow::Component
-  input_port :in
-  output_port :out
-  output_port :errored
-
-  def process_message(input_port, input_port_key, connection, message)
-    out.each do |connections|
-      begin
-        connections.send_message message
-      rescue Exception => e
-        RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
-        errored.send_message message
-      end
-    end
-  end
-end
-
-
-class RFlow::Components::RubyProcFilter < RFlow::Component
-  input_port :in
-  output_port :filtered
-  output_port :dropped
-  output_port :errored
-
-
-  def configure!(config)
-    @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
-  end
-
-  def process_message(input_port, input_port_key, connection, message)
-    RFlow.logger.debug "Filtering message"
-    begin
-      if @filter_proc.call(message)
-        filtered.send_message message
-      else
-        dropped.send_message message
-      end
-    rescue Exception => e
-      RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
-      errored.send_message message
-    end
-  end
-end
-
+require 'eventmachine'
+require 'evma_httpserver'
 
 class RFlow::Components::FileOutput < RFlow::Component
   attr_accessor :output_file_path, :output_file
@@ -67,8 +23,6 @@ class RFlow::Components::FileOutput < RFlow::Component
   end
 end
 
-# TODO: Ensure that all the following methods work as they are
-# supposed to.  This is the interface that I'm adhering to
 class SimpleComponent < RFlow::Component
   input_port :in
   output_port :out
@@ -80,7 +34,6 @@ class SimpleComponent < RFlow::Component
   def cleanup!; end
 end
 
-
 http_request_schema =<<EOS
 {
     "type": "record",
@@ -108,7 +61,6 @@ http_response_schema =<<EOS
 EOS
 RFlow::Configuration.add_available_data_type('HTTPResponse', 'avro', http_response_schema)
 
-
 # Need to be careful when extending to not clobber data already in data_object
 module HTTPRequestExtension
   def self.extended(base_data)
@@ -120,7 +72,6 @@ module HTTPRequestExtension
 end
 RFlow::Configuration.add_available_data_extension('HTTPRequest', HTTPRequestExtension)
 
-
 # Need to be careful when extending to not clobber data already in data_object
 module HTTPResponseExtension
   def self.extended(base_data)
@@ -135,10 +86,6 @@ module HTTPResponseExtension
 end
 RFlow::Configuration.add_available_data_extension('HTTPResponse', HTTPResponseExtension)
 
-
-require 'eventmachine'
-require 'evma_httpserver'
-
 class HTTPServer < RFlow::Component
   input_port :response_port
   output_port :request_port
@@ -148,7 +95,7 @@ class HTTPServer < RFlow::Component
   def configure!(config)
     @listen = config['listen'] ? config['listen'] : '127.0.0.1'
     @port = config['port'] ? config['port'].to_i : 8000
-    @connections = Hash.new
+    @connections = {}
   end
 
   def run!
@@ -169,6 +116,7 @@ class HTTPServer < RFlow::Component
     RFlow.logger.debug "Received a HTTPResponse message, determining if its mine"
     my_events = message.provenance.find_all {|processing_event| processing_event.component_instance_uuid == instance_uuid}
     RFlow.logger.debug "Found #{my_events.size} processing events from me"
+
     # Attempt to send the data to each context match
     my_events.each do |processing_event|
       RFlow.logger.debug "Inspecting #{processing_event.context}"
@@ -192,13 +140,11 @@ class HTTPServer < RFlow::Component
       no_environment_strings
     end
 
-
     def receive_data(data)
       RFlow.logger.debug "Received #{data.bytesize} data from #{client_ip}:#{client_port}"
       super
     end
 
-
     def process_http_request
       RFlow.logger.debug "Received a full HTTP request from #{client_ip}:#{client_port}"
 
@@ -214,8 +160,7 @@ class HTTPServer < RFlow::Component
       server.request_port.send_message request_message
     end
 
-
-    def send_http_response(response_message=nil)
+    def send_http_response(response_message = nil)
       RFlow.logger.debug "Sending an HTTP response to #{client_ip}:#{client_port}"
       resp = EventMachine::DelegatedHttpResponse.new(self)
 
@@ -234,11 +179,10 @@ class HTTPServer < RFlow::Component
       close_connection_after_writing
     end
 
-
     # Called when a connection is torn down for whatever reason.
     # Remove this connection from the server's list
     def unbind
-      RFlow.logger.debug "Connection to lost"
+      RFlow.logger.debug "Connection lost"
       server.connections.delete(self.signature)
     end
   end
@@ -259,4 +203,3 @@ class HTTPResponder < RFlow::Component
     response.send_message response_message
   end
 end
-