rflow 0.0.5

data/example/basic_config.rb

@@ -0,0 +1,49 @@
+# Meat of the config file.  Stuff above this should probably be in
+# separate gems and/or files that are brought in at runtime.
+RFlow::Configuration::RubyDSL.configure do |config|
+  # Configure the settings, which include paths for various files, log
+  # levels, and component specific stuffs
+  config.setting('rflow.log_level', 'DEBUG')
+  config.setting('rflow.application_directory_path', '.')
+
+  # Add schemas to the list of available.  Not convinced this is necessary
+#  config.schema('schemaname', 'schematype', 'schemadata')
+
+  # Instantiate components
+#  config.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3, 'interval_seconds' => 1
+#  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+#  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'lambda {|message| true}'
+#  config.component 'replicate', 'RFlow::Components::Replicate'
+#  config.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap1'
+#  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap2'
+  
+  # Hook components together
+  # config.connect 'generate_ints#out' => 'filter#in'
+  # config.connect 'filter#filtered' => 'replicate#in'
+  # config.connect 'replicate#out[0]' => 'simple#in'
+  # config.connect 'replicate#out[one]' => 'complex#in'
+  # config.connect 'simple#out' => 'output#in'
+  # config.connect 'complex#out' => 'output#in'
+
+#  config.connect 'generate_ints1#out' => 'filter#in'
+#  config.connect 'generate_ints2#out' => 'filter#in'
+#  config.connect 'filter#filtered' => 'replicate#in'
+#  config.connect 'replicate#out[1]' => 'output1#in'
+#  config.connect 'replicate#out[2]' => 'output2#in'
+  # Some tests that should fail
+  # output should not have an 'out' ports
+#  config.connect 'output#out' => 'simple#in'
+
+  config.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+  config.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap'
+  config.component 'output_even', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap_even'
+  config.component 'output_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap_odd'
+
+  config.connect 'generate_ints#even_odd_out' => 'output#in'
+  config.connect 'generate_ints#even_odd_out[even]' => 'output_even#in'
+  config.connect 'generate_ints#even_odd_out[odd]' => 'output_odd#in'
+  
+
+end
+
+

data/example/basic_extensions.rb

@@ -0,0 +1,142 @@
+# This will/should bring in available components and their schemas
+require 'rflow/components'
+require 'rflow/message'
+
+#RFlow::Configuration.add_available_data_schema RFlow::Message::Data::AvroSchema.new('Integer', long_integer_schema)
+
+# Example of creating and registering a data extension
+module SimpleDataExtension
+  # Use this to default/verify the data in data_object
+  def self.extended(base_data)
+    base_data.data_object
+  end
+
+  def my_method; end
+end
+RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Integer', SimpleDataExtension)
+
+
+
+# Example of creating and registering a new schema
+long_integer_schema = '{"type": "long"}'
+RFlow::Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', long_integer_schema)
+
+
+class RFlow::Components::GenerateIntegerSequence < RFlow::Component
+  output_port :out
+  output_port :even_odd_out
+  
+  def configure!(config)
+    @start = config['start'].to_i
+    @finish = config['finish'].to_i
+    @step = config['step'] ? config['step'].to_i : 1
+    # If interval seconds is not given, it will default to 0
+    @interval_seconds = config['interval_seconds'].to_i
+  end
+
+  # Note that this uses the timer (sometimes with 0 interval) so as
+  # not to block the reactor
+  def run!
+    timer = EM::PeriodicTimer.new(@interval_seconds) do 
+      message = RFlow::Message.new('RFlow::Message::Data::Integer')
+      message.data.data_object = @start
+      out.send_message message
+      if @start % 2 == 0
+        even_odd_out['even'].send_message message
+      else
+        even_odd_out['odd'].send_message message
+      end
+      even_odd_out.send_message message
+      
+      @start += @step
+      timer.cancel if @start > @finish
+    end
+  end
+
+end
+
+class RFlow::Components::Replicate < RFlow::Component
+  input_port :in
+  output_port :out
+  output_port :errored
+  
+  def process_message(input_port, input_port_key, connection, message)
+    puts "Processing message in Replicate"
+    out.each do |connections|
+      puts "Replicating"
+      begin
+        connections.send_message message
+      rescue Exception => e
+        puts "Exception #{e.message}"
+        errored.send_message message
+      end
+    end
+  end
+end
+
+puts "Before RubyProcFilter"
+class RFlow::Components::RubyProcFilter < RFlow::Component
+  input_port :in
+  output_port :filtered
+  output_port :dropped
+  output_port :errored
+
+
+  def configure!(config)
+    @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
+  end
+  
+  def process_message(input_port, input_port_key, connection, message)
+    puts "Processing message in RubyProcFilter"
+    begin
+      if @filter_proc.call(message)
+        filtered.send_message message
+      else
+        dropped.send_message message
+      end
+    rescue Exception => e
+      puts "Attempting to send message to errored #{e.message}"
+      errored.send_message message
+    end
+  end
+end
+
+puts "Before FileOutput"
+class RFlow::Components::FileOutput < RFlow::Component
+  attr_accessor :output_file_path, :output_file
+  input_port :in
+
+  def configure!(config)
+    self.output_file_path = config['output_file_path']
+    self.output_file = File.new output_file_path, 'w+'
+  end
+
+  #def run!; end
+  
+  def process_message(input_port, input_port_key, connection, message)
+    puts "About to output to a file #{output_file_path}"
+    output_file.puts message.data.data_object.inspect
+    output_file.flush
+  end
+
+  
+  def cleanup
+    output_file.close
+  end
+  
+end
+
+# TODO: Ensure that all the following methods work as they are
+# supposed to.  This is the interface that I'm adhering to
+class SimpleComponent < RFlow::Component
+  input_port :in
+  output_port :out
+
+  def configure!(config); end
+  def run!; end
+  def process_message(input_port, input_port_key, connection, message); end
+  def shutdown!; end
+  def cleanup!; end
+end
+
+

data/example/http_config.rb

@@ -0,0 +1,21 @@
+RFlow::Configuration::RubyDSL.configure do |config|
+  # Configure the settings, which include paths for various files, log
+  # levels, and component specific stuffs
+  config.setting('rflow.log_level', 'INFO')
+  config.setting('rflow.application_directory_path', '.')
+
+  # Instantiate components
+  config.component 'http_server', 'HTTPServer', 'port' => 8080
+  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'message.data.path == "/awesome"'
+  config.component 'replicate', 'RFlow::Components::Replicate'
+  config.component 'file_output', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/http_crap'
+  config.component 'http_responder', 'HTTPResponder', 'response_code' => 200, 'content' => 'Hi, this teh awesome'
+  
+#  config.connect 'http_server#request_port' => 'filter#in'
+#  config.connect 'filter#filtered' => 'replicate#in'
+  config.connect 'http_server#request_port' => 'replicate#in'
+  config.connect 'replicate#out[1]' => 'file_output#in'
+  config.connect 'replicate#out[2]' => 'http_responder#request'
+  config.connect 'http_responder#response' => 'http_server#response_port'
+end
+

data/example/http_extensions.rb

@@ -0,0 +1,262 @@
+# This will/should bring in available components and their schemas
+require 'rflow/components'
+require 'rflow/message'
+
+
+class RFlow::Components::Replicate < RFlow::Component
+  input_port :in
+  output_port :out
+  output_port :errored
+  
+  def process_message(input_port, input_port_key, connection, message)
+    out.each do |connections|
+      begin
+        connections.send_message message
+      rescue Exception => e
+        RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
+        errored.send_message message
+      end
+    end
+  end
+end
+
+
+class RFlow::Components::RubyProcFilter < RFlow::Component
+  input_port :in
+  output_port :filtered
+  output_port :dropped
+  output_port :errored
+
+
+  def configure!(config)
+    @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
+  end
+  
+  def process_message(input_port, input_port_key, connection, message)
+    RFlow.logger.debug "Filtering message"
+    begin
+      if @filter_proc.call(message)
+        filtered.send_message message
+      else
+        dropped.send_message message
+      end
+    rescue Exception => e
+      RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
+      errored.send_message message
+    end
+  end
+end
+
+
+class RFlow::Components::FileOutput < RFlow::Component
+  attr_accessor :output_file_path, :output_file
+  input_port :in
+
+  def configure!(config)
+    self.output_file_path = config['output_file_path']
+    self.output_file = File.new output_file_path, 'w+'
+  end
+
+  def process_message(input_port, input_port_key, connection, message)
+    output_file.puts message.data.data_object.inspect
+    output_file.flush
+  end
+  
+  def cleanup!
+    output_file.close
+  end
+end
+
+# TODO: Ensure that all the following methods work as they are
+# supposed to.  This is the interface that I'm adhering to
+class SimpleComponent < RFlow::Component
+  input_port :in
+  output_port :out
+
+  def configure!(config); end
+  def run!; end
+  def process_message(input_port, input_port_key, connection, message); end
+  def shutdown!; end
+  def cleanup!; end
+end
+
+
+http_request_schema =<<EOS
+{
+    "type": "record",
+    "name": "HTTPRequest",
+    "namespace": "org.rflow",
+    "aliases": [],
+    "fields": [
+        {"name": "path", "type": "string"}
+    ]
+}
+EOS
+RFlow::Configuration.add_available_data_type('HTTPRequest', 'avro', http_request_schema)
+
+http_response_schema =<<EOS
+{
+    "type": "record",
+    "name": "HTTPResponse",
+    "namespace": "org.rflow",
+    "aliases": [],
+    "fields": [
+        {"name": "status", "type": "int"},
+        {"name": "content", "type": "bytes"}
+    ]
+}
+EOS
+RFlow::Configuration.add_available_data_type('HTTPResponse', 'avro', http_response_schema)
+
+
+# Need to be careful when extending to not clobber data already in data_object
+module HTTPRequestExtension 
+  def self.extended(base_data)
+    base_data.data_object ||= {'path' => ''}
+  end
+
+  def path; data_object['path']; end
+  def path=(new_path); data_object['path'] = new_path; end
+end
+RFlow::Configuration.add_available_data_extension('HTTPRequest', HTTPRequestExtension)
+
+
+# Need to be careful when extending to not clobber data already in data_object
+module HTTPResponseExtension 
+  def self.extended(base_data)
+    base_data.data_object ||= {'status' => 200, 'content' => ''}
+  end
+
+  def status; data_object['status']; end
+  def status=(new_status); data_object['status'] = new_status; end
+  
+  def content; data_object['content']; end
+  def content=(new_content); data_object['content'] = new_content; end
+end
+RFlow::Configuration.add_available_data_extension('HTTPResponse', HTTPResponseExtension)
+
+
+require 'eventmachine'
+require 'evma_httpserver'
+
+class HTTPServer < RFlow::Component
+  input_port :response_port
+  output_port :request_port
+
+  attr_accessor :port, :listen, :server_signature, :connections
+  
+  def configure!(config)
+    @listen = config['listen'] ? config['listen'] : '127.0.0.1'
+    @port = config['port'] ? config['port'].to_i : 8000
+    @connections = Hash.new
+  end
+
+  def run!
+    @server_signature = EM.start_server(@listen, @port, Connection) do |conn|
+      conn.server = self
+      self.connections[conn.signature.to_s] = conn
+    end
+  end
+
+  # Getting all messages to response_port, which we need to filter for
+  # those that pertain to this component and have active connections.
+  # This is done by inspecting the provenance, specifically the
+  # context attribute that we stored originally
+  def process_message(input_port, input_port_key, connection, message)
+    RFlow.logger.debug "Received a message"
+    return unless message.data_type_name == 'HTTPResponse'
+
+    RFlow.logger.debug "Received a HTTPResponse message, determining if its mine"
+    my_events = message.provenance.find_all {|processing_event| processing_event.component_instance_uuid == instance_uuid}
+    RFlow.logger.debug "Found #{my_events.size} processing events from me"
+    # Attempt to send the data to each context match
+    my_events.each do |processing_event|
+      RFlow.logger.debug "Inspecting #{processing_event.context}"
+      ip, port, connection_signature = processing_event.context.split ':'
+      if connections[connection_signature]
+        RFlow.logger.debug "Found connection for #{processing_event.context}"
+        connections[connection_signature].send_http_response message
+      end
+    end
+  end
+  
+  class Connection < EventMachine::Connection
+    include EventMachine::HttpServer
+
+    attr_accessor :server, :client_ip, :client_port
+
+    def post_init
+      @client_port, @client_ip = Socket.unpack_sockaddr_in(get_peername) rescue ["?", "?.?.?.?"]
+      RFlow.logger.debug "Connection from #{@client_ip}:#{@client_port}"
+      super
+      no_environment_strings
+    end
+
+    
+    def receive_data(data)
+      RFlow.logger.debug "Received #{data.bytesize} data from #{client_ip}:#{client_port}"
+      super
+    end
+      
+    
+    def process_http_request
+      RFlow.logger.debug "Received a full HTTP request from #{client_ip}:#{client_port}"
+      
+      processing_event = RFlow::Message::ProcessingEvent.new(server.instance_uuid, Time.now.utc)
+
+      request_message = RFlow::Message.new('HTTPRequest')
+      request_message.data.path = @http_request_uri
+
+      processing_event.context = "#{client_ip}:#{client_port}:#{signature}"
+      processing_event.completed_at = Time.now.utc
+      request_message.provenance << processing_event
+
+      server.request_port.send_message request_message
+    end
+
+    
+    def send_http_response(response_message=nil)
+      RFlow.logger.debug "Sending an HTTP response to #{client_ip}:#{client_port}"
+      resp = EventMachine::DelegatedHttpResponse.new(self)
+
+      # Default values
+      resp.status                  = 200
+      resp.content                 = ""
+      resp.headers["Content-Type"] = "text/html; charset=UTF-8"
+      resp.headers["Server"]       = "Apache/2.2.3 (CentOS)"
+      
+      if response_message
+        resp.status  = response_message.data.status
+        resp.content = response_message.data.content
+      end
+
+      resp.send_response
+      close_connection_after_writing
+    end
+
+    
+    # Called when a connection is torn down for whatever reason.
+    # Remove this connection from the server's list
+    def unbind
+      RFlow.logger.debug "Connection to lost"
+      server.connections.delete(self.signature)
+    end
+  end
+end
+
+# As this component creates a new message type based on another, it
+# copies over the provenance.  It does not bother to add its own
+# processing_event to the provenance, but it could/should
+class HTTPResponder < RFlow::Component
+  input_port :request
+  output_port :response
+
+  def process_message(input_port, input_port_key, connection, message)
+    response_message = RFlow::Message.new('HTTPResponse')
+    response_message.data.status = 404
+    response_message.data.content = "CONTENT: #{message.data.path} was accessed"
+    response_message.provenance = message.provenance
+    response.send_message response_message
+  end
+end
+