rflow 0.0.5

data/schema/message.avsc

@@ -0,0 +1,36 @@
+{
+    "type": "record",
+    "name": "Message",
+    "namespace": "org.rflow",
+    "aliases": [],
+    "fields": [
+        {"name": "data_type_name", "type": "string"},
+        {"name": "provenance", 
+         "type": {
+             "type": "array", 
+             "items": {
+                 "type": "record", 
+                 "name": "ProcessingEvent", 
+                 "namespace": "org.rflow",
+                 "aliases": [],
+                 "fields": [
+                     {"name": "component_instance_uuid", "type": "string"},
+                     {"name": "started_at", "type": ["string", "null"]},
+                     {"name": "completed_at", "type": ["string", "null"]},
+                     {"name": "context", "type": ["bytes", "null"]}
+                 ]
+             }
+         }
+        },
+        {"name": "data_serialization_type", 
+         "type": {
+             "type": "enum", 
+             "name": "DataSerializationType", 
+             "symbols": ["avro", "xml"]
+         }
+        },
+        {"name": "data_schema", "type": "string"},
+        {"name": "data", "type": "bytes"}
+    ]
+}
+    

data/schema/raw.avsc

@@ -0,0 +1,9 @@
+{
+    "type": "record",
+    "name": "Raw",
+    "namespace": "org.rflow.message.data",
+    "aliases": [],
+    "fields": [
+        {"name": "raw", "type": "bytes"}
+    ]
+}

data/spec/fixtures/config_ints.rb

@@ -0,0 +1,61 @@
+# Meat of the config file.  Stuff above this should probably be in
+# separate gems and/or files that are brought in at runtime.
+RFlow::Configuration::RubyDSL.configure do |config|
+  # Configure the settings, which include paths for various files, log
+  # levels, and component specific stuffs
+  config.setting('rflow.log_level', 'DEBUG')
+  config.setting('rflow.application_directory_path', '../tmp')
+
+  config.setting('rflow.application_name', 'testapp')
+  
+  # Add schemas to the list of available.  Not convinced this is necessary
+#  config.schema('schemaname', 'schematype', 'schemadata')
+
+  # Instantiate components
+#  config.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', 'start' => 0, 'finish' => 10, 'step' => 3, 'interval_seconds' => 1
+#  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+#  config.component 'filter', 'RFlow::Components::RubyProcFilter', 'filter_proc_string' => 'lambda {|message| true}'
+#  config.component 'replicate', 'RFlow::Components::Replicate'
+#  config.component 'output1', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap1'
+#  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '/tmp/crap2'
+  
+  # Hook components together
+  # config.connect 'generate_ints#out' => 'filter#in'
+  # config.connect 'filter#filtered' => 'replicate#in'
+  # config.connect 'replicate#out[0]' => 'simple#in'
+  # config.connect 'replicate#out[one]' => 'complex#in'
+  # config.connect 'simple#out' => 'output#in'
+  # config.connect 'complex#out' => 'output#in'
+
+#  config.connect 'generate_ints1#out' => 'filter#in'
+#  config.connect 'generate_ints2#out' => 'filter#in'
+#  config.connect 'filter#filtered' => 'replicate#in'
+#  config.connect 'replicate#out[1]' => 'output1#in'
+#  config.connect 'replicate#out[2]' => 'output2#in'
+  # Some tests that should fail
+  # output should not have an 'out' ports
+#  config.connect 'output#out' => 'simple#in'
+
+  config.component 'generate_ints', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+  config.component 'output', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out'
+  config.component 'output2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out2'
+  config.component 'output_even', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even'
+  config.component 'output_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_odd'
+  config.component 'output_even_odd', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd'
+
+  config.connect 'generate_ints#out' => 'output#in'
+  config.connect 'generate_ints#out' => 'output2#in'
+  config.connect 'generate_ints#even_odd_out[even]' => 'output_even#in'
+  config.connect 'generate_ints#even_odd_out[odd]' => 'output_odd#in'
+  config.connect 'generate_ints#even_odd_out' => 'output_even_odd#in'
+
+
+  
+  config.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', 'start' => 20, 'finish' => 30
+  config.component 'output_even_odd2', 'RFlow::Components::FileOutput', 'output_file_path' => '../tmp/out_even_odd2'
+  
+  config.connect 'generate_ints2#even_odd_out' => 'output_even_odd2#in'
+
+end
+
+

data/spec/fixtures/extensions_ints.rb

@@ -0,0 +1,141 @@
+# This will/should bring in available components and their schemas
+require 'rflow/components'
+require 'rflow/message'
+
+#RFlow::Configuration.add_available_data_schema RFlow::Message::Data::AvroSchema.new('Integer', long_integer_schema)
+
+# Example of creating and registering a data extension
+module SimpleDataExtension
+  # Use this to default/verify the data in data_object
+  def self.extended(base_data)
+    base_data.data_object
+  end
+
+  def my_method; end
+end
+RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Integer', SimpleDataExtension)
+
+
+
+# Example of creating and registering a new schema
+long_integer_schema = '{"type": "long"}'
+RFlow::Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', long_integer_schema)
+
+
+class RFlow::Components::GenerateIntegerSequence < RFlow::Component
+  output_port :out
+  output_port :even_odd_out
+  
+  def configure!(config)
+    @start = config['start'].to_i
+    @finish = config['finish'].to_i
+    @step = config['step'] ? config['step'].to_i : 1
+    # If interval seconds is not given, it will default to 0
+    @interval_seconds = config['interval_seconds'].to_i
+  end
+
+  # Note that this uses the timer (sometimes with 0 interval) so as
+  # not to block the reactor
+  def run!
+    timer = EM::PeriodicTimer.new(@interval_seconds) do 
+      message = RFlow::Message.new('RFlow::Message::Data::Integer')
+      message.data.data_object = @start
+      out.send_message message
+      if @start % 2 == 0
+        even_odd_out['even'].send_message message
+      else
+        even_odd_out['odd'].send_message message
+      end
+      
+      @start += @step
+      timer.cancel if @start > @finish
+    end
+  end
+
+end
+
+class RFlow::Components::Replicate < RFlow::Component
+  input_port :in
+  output_port :out
+  output_port :errored
+  
+  def process_message(input_port, input_port_key, connection, message)
+    puts "Processing message in Replicate"
+    out.each do |connections|
+      puts "Replicating"
+      begin
+        connections.send_message message
+      rescue Exception => e
+        puts "Exception #{e.message}"
+        errored.send_message message
+      end
+    end
+  end
+end
+
+puts "Before RubyProcFilter"
+class RFlow::Components::RubyProcFilter < RFlow::Component
+  input_port :in
+  output_port :filtered
+  output_port :dropped
+  output_port :errored
+
+
+  def configure!(config)
+    @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
+  end
+  
+  def process_message(input_port, input_port_key, connection, message)
+    puts "Processing message in RubyProcFilter"
+    begin
+      if @filter_proc.call(message)
+        filtered.send_message message
+      else
+        dropped.send_message message
+      end
+    rescue Exception => e
+      puts "Attempting to send message to errored #{e.message}"
+      errored.send_message message
+    end
+  end
+end
+
+puts "Before FileOutput"
+class RFlow::Components::FileOutput < RFlow::Component
+  attr_accessor :output_file_path, :output_file
+  input_port :in
+
+  def configure!(config)
+    self.output_file_path = config['output_file_path']
+    self.output_file = File.new output_file_path, 'w+'
+  end
+
+  #def run!; end
+  
+  def process_message(input_port, input_port_key, connection, message)
+    puts "About to output to a file #{output_file_path}"
+    output_file.puts message.data.data_object.inspect
+    output_file.flush
+  end
+
+  
+  def cleanup
+    output_file.close
+  end
+  
+end
+
+# TODO: Ensure that all the following methods work as they are
+# supposed to.  This is the interface that I'm adhering to
+class SimpleComponent < RFlow::Component
+  input_port :in
+  output_port :out
+
+  def configure!(config); end
+  def run!; end
+  def process_message(input_port, input_port_key, connection, message); end
+  def shutdown!; end
+  def cleanup!; end
+end
+
+

data/spec/rflow_configuration_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper.rb'
+
+require 'rflow/configuration'
+
+
+describe RFlow::Configuration do
+  before(:each) do
+#    RFlow::Configuration.available_data_types.clear
+#    RFlow::Configuration.available_data_extensions.clear
+  end
+
+  
+  describe '.add_available_data_type' do
+    context 'if passed a data_serialization that is not avro or xml' do
+      it "should throw an exception" do
+        expect do
+          RFlow::Configuration.add_available_data_type('A', 'boom', 'schema')
+        end.to raise_error(ArgumentError)
+      end
+
+      it "should not update the available_data_types" do
+        num_types = RFlow::Configuration.available_data_types.size
+        RFlow::Configuration.add_available_data_type('A', 'boom', 'schema') rescue nil
+        RFlow::Configuration.available_data_types.should have(num_types).items
+      end
+    end
+  end
+
+  describe "Data Extensions" do
+  
+    describe ".add_available_data_extension" do
+      context 'if passed a non-module data extension' do
+        it "should throw an exception" do
+          expect do
+            RFlow::Configuration.add_available_data_extension('data_type', 'not a Module')
+          end.to raise_error(ArgumentError)
+        end
+      end
+      
+      context "if passed a valid Module as a data extension" do
+        it "should update the available_data_extensions" do
+          num_extensions = RFlow::Configuration.available_data_extensions['data_type'].size
+          expect do
+            RFlow::Configuration.add_available_data_extension('data_type', Module.new)
+          end.to_not raise_error
+          RFlow::Configuration.available_data_extensions['data_type'].should have(num_extensions+1).items
+        end
+      end
+    end
+    
+    it "should perform simple 'prefix'-based inheritance for extensions" do
+      RFlow::Configuration.add_available_data_extension('A', A = Module.new)
+      RFlow::Configuration.add_available_data_extension('A::B', B = Module.new)
+      RFlow::Configuration.add_available_data_extension('A::B::C', C = Module.new)
+      RFlow::Configuration.add_available_data_extension('A::B::C::D', D = Module.new)
+
+      RFlow::Configuration.available_data_extensions['A'].should have(1).item
+      RFlow::Configuration.available_data_extensions['A'].should == [A]
+
+      RFlow::Configuration.available_data_extensions['A::B'].should have(2).item
+      RFlow::Configuration.available_data_extensions['A::B'].should == [A, B]
+
+      RFlow::Configuration.available_data_extensions['A::B::C'].should have(3).item
+      RFlow::Configuration.available_data_extensions['A::B::C'].should == [A, B, C]
+
+      RFlow::Configuration.available_data_extensions['A::B::C::D'].should have(4).item
+      RFlow::Configuration.available_data_extensions['A::B::C::D'].should == [A, B, C, D]
+
+      RFlow::Configuration.available_data_extensions['A::B::C::D::E'].should have(4).item
+      RFlow::Configuration.available_data_extensions['A::B::C::D::E'].should == [A, B, C, D]
+    end
+  end
+end

data/spec/rflow_message_data_raw.rb

@@ -0,0 +1,26 @@
+require 'spec_helper.rb'
+
+require 'rflow/components/raw'
+
+describe 'RFlow::Message::Data::Raw Avro Schema' do
+  before(:each) do 
+    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
+  end
+
+  it "should load the schema" do
+    @schema_string.should_not == nil
+  end
+  
+  it "should encode and decode an object" do
+    raw = {'raw' => 'rawdata'}
+
+    expect {encode_avro(@schema_string, raw)}.to_not raise_error
+    avro_encoded_raw = encode_avro(@schema_string, raw)
+
+    expect {decode_avro(@schema_string, avro_encoded_raw)}.to_not raise_error
+    decoded_raw = decode_avro(@schema_string, avro_encoded_raw)
+
+    decoded_raw.should == raw
+  end
+
+end

data/spec/rflow_message_data_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper.rb'
+
+require 'rflow/message'
+
+describe RFlow::Message::Data do
+  before(:all) do
+    @string = 'this is a string to be serialized'
+    @invalid_avro_schema_string = 'invalid schema'
+    @valid_avro_string_schema_string = '{"type": "string"}'
+    @avro_serialized_string = encode_avro(@valid_avro_string_schema_string, @string)
+  end
+
+  context "if created without a schema" do
+    it "should throw an exception" do
+      expect {RFlow::Message::Data.new()}.to raise_error(ArgumentError)
+    end
+  end
+
+  context "if created with an invalid schema for the serialization" do
+    it "should throw and exception" do 
+      expect {RFlow::Message::Data.new(@invalid_avro_schema_string)}.to raise_error(ArgumentError)
+      expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
+      expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
+    end
+  end
+  
+  context "if created with a valid avro schema and serialization" do
+  end
+
+  context "if created with a valid avro schema" do 
+    it "should instantiate correctly" do 
+      expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
+      expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
+    end
+    
+    context "if created with a non-avro data serialization" do
+      it "should throw an exception" do
+        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'unknown')}.to raise_error(ArgumentError)
+        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :unknown)}.to raise_error(ArgumentError)
+        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'xml')}.to raise_error(ArgumentError)
+        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :xml)}.to raise_error(ArgumentError)
+      end
+    end
+    
+    context "if created with an avro serialization" do
+      it "should instantiate correctly" do
+        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
+        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
+      end
+
+      context "if created with a serialized data object" do
+        it "should instantiate correctly" do
+          expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro', @avro_serialized_string)}.to_not raise_error
+          message = RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro', @avro_serialized_string)
+          p message
+        end
+      end
+    end
+  end
+end

data/spec/rflow_message_spec.rb

@@ -0,0 +1,182 @@
+require 'spec_helper.rb'
+
+require 'digest/md5'
+
+require 'rflow/message'
+
+describe RFlow::Message do
+
+  context "if created with an unknown data type" do
+    it "should throw an exception" do
+      expect {RFlow::Message.new('non_existant_data_type')}.to raise_error(ArgumentError)
+    end
+  end
+
+
+  context "if created with a known data type" do
+    before(:all) do
+      @avro_string_schema_string = '{"type": "string"}'
+      RFlow::Configuration.add_available_data_type(:string_type, 'avro', @avro_string_schema_string)
+    end
+
+    it "should instantiate correctly" do
+      expect {RFlow::Message.new('string_type')}.to_not raise_error
+    end
+
+    context "if created with empty provenance" do
+      context "if created with an unknown data serialization" do
+        it "should throw an exception" do
+          expect {RFlow::Message.new('string_type', [], 'unknown')}.to raise_error(ArgumentError)
+          expect {RFlow::Message.new('string_type', [], :unknown)}.to raise_error(ArgumentError)
+        end
+      end
+
+      context "if created with a known data serialization" do
+        it "should instantiate correctly" do
+          expect {RFlow::Message.new('string_type', [], 'avro')}.to_not raise_error
+          expect {RFlow::Message.new('string_type', [], 'avro')}.to_not raise_error
+        end
+
+        context "if created with a mismatched schema" do
+        end
+
+        context "if created with a matched schema" do
+        end
+
+
+        context "if created with a nil schema" do
+          context "if created with a serialized data object" do
+            before(:all) do
+              @string = 'this is a string to be serialized'
+              @avro_serialized_string = encode_avro(@avro_string_schema_string, @string)
+            end
+            
+            it "should instantiate correctly" do
+              expect {RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)}.to_not raise_error
+              message = RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)
+            end
+          end
+        end
+      end
+    end
+
+    context "if created with invalid provenance" do
+      before(:all) do
+        @invalid_processing_event_hash = {'started_at' => 'bad time string'}
+        @invalid_provenance = [@invalid_processing_event_hash]
+      end
+      
+      it "should throw an exception" do
+        expect {RFlow::Message.new('string_type', @invalid_provenance)}.to raise_error(ArgumentError)
+      end
+    end
+    
+    context "if created with valid provenance" do
+      before(:all) do
+        @valid_xmlschema_time = '2001-01-01T01:01:01.000001Z'
+        @valid_processing_event_hash = {'component_instance_uuid' => 'uuid', 'started_at' => @valid_xmlschema_time}
+        @valid_processing_event = RFlow::Message::ProcessingEvent.new('uuid', @valid_xmlschema_time, @valid_xmlschema_time, 'context')
+        @valid_provenance = [
+                             RFlow::Message::ProcessingEvent.new('uuid'),
+                             @valid_processing_event_hash,
+                             @valid_processing_event,
+                            ]
+        @valid_provenance_hashes = [
+                                    {"component_instance_uuid"=>"uuid", "started_at"=>nil, "completed_at"=>nil, "context"=>nil},
+                                    {"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>nil, "context"=>nil},
+                                    {"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>@valid_xmlschema_time, "context"=>"context"},
+                                   ]
+      end
+      
+      it "should instantiate correctly" do
+        p @valid_provenance
+        expect {RFlow::Message.new('string_type', @valid_provenance)}.to_not raise_error
+      end
+
+      it "should correctly set the provenance processing events" do
+        message = RFlow::Message.new('string_type', @valid_provenance)
+        message.provenance[1].component_instance_uuid.should == 'uuid'
+        message.provenance[1].started_at.should == Time.xmlschema(@valid_xmlschema_time)
+        message.provenance[1].completed_at.should == nil
+        message.provenance[1].context.should == nil
+      end
+
+      it "should to_hash its provenance correctly" do
+        message = RFlow::Message.new('string_type', @valid_provenance)
+        message.provenance.map(&:to_hash).should == @valid_provenance_hashes
+      end
+      
+    end
+
+    context "if correctly created" do
+      it "should serialize and deserialized correctly to/from avro" do
+        message = RFlow::Message.new('string_type')
+        message.provenance << RFlow::Message::ProcessingEvent.new('UUID')
+        message.data.data_object = 'teh awesome'
+
+        processed_message = RFlow::Message.from_avro(message.to_avro)
+        message.data.to_avro.should == processed_message.data.to_avro
+        message.data.data_object.should == processed_message.data.data_object
+      end
+    end
+    
+    context "if data extensions exist" do
+      it "should extend the data element with the extension" do
+        module ExtensionModule; def ext_method; end; end
+
+        message = RFlow::Message.new('string_type')
+        message.data.methods.should_not include(:ext_method)
+
+        RFlow::Configuration.add_available_data_extension('string_type', ExtensionModule)
+        message = RFlow::Message.new('string_type')
+        message.data.methods.should include(:ext_method)
+
+      end
+    end
+  end
+
+  it "should correctly handle large raw types" do
+    message = RFlow::Message.new('RFlow::Message::Data::Raw')
+    message.data.raw = Array.new(101) { rand(256) }.pack('c*')
+    
+    message_avro = message.to_avro.force_encoding('BINARY')
+    
+    processed_message = RFlow::Message.from_avro(message_avro)
+    processed_message_avro = processed_message.to_avro.force_encoding('BINARY')
+
+    @raw_schema = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
+
+    encode_avro(@raw_schema, message.data.data_object).should == message.data.to_avro
+    decode_avro(@raw_schema, message.data.to_avro).should == message.data.data_object
+    
+    p message.data.raw
+    p message_avro
+    p message_avro.bytesize
+    p processed_message_avro
+    p processed_message_avro.bytesize
+    
+    p message_avro.encoding
+    p message_avro.valid_encoding?
+
+    p processed_message_avro.encoding
+    p processed_message_avro.valid_encoding?
+
+    message_data_avro = message.data.to_avro.force_encoding('BINARY')
+    processed_message_data_avro = processed_message.data.to_avro.force_encoding('BINARY')
+    
+    p message_data_avro.encoding
+    p message_data_avro.valid_encoding?
+    p message_data_avro
+    p processed_message_data_avro.encoding
+    p processed_message_data_avro.valid_encoding?
+    p processed_message_data_avro
+
+    Digest::MD5.hexdigest(message_avro).should == Digest::MD5.hexdigest(processed_message_avro)
+
+    
+    message_data_avro.should == processed_message_data_avro
+    Digest::MD5.hexdigest(message_data_avro).should == Digest::MD5.hexdigest(processed_message_data_avro)
+    Digest::MD5.hexdigest(message.data.raw).should == Digest::MD5.hexdigest(processed_message.data.raw)
+  end
+  
+end