rflow 1.0.0a1 → 1.0.0a2

data/spec/rflow/components/clock_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+class RFlow
+  module Components
+    describe Clock do
+      before(:each) do
+        ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+        Configuration.migrate_database
+      end
+      let(:config) do
+        RFlow::Configuration::Component.new.tap do |c|
+          c.output_ports << RFlow::Configuration::OutputPort.new(name: 'tick_port')
+        end
+      end
+      let(:message_connection) { RFlow::MessageCollectingConnection.new }
+
+      def clock(args = {})
+        Clock.new(config).tap do |c|
+          c.configure! args
+          c.tick_port.connect!
+          c.tick_port.add_connection nil, message_connection
+        end
+      end
+
+      def messages; message_connection.messages; end
+
+      it 'defaults configuration nicely' do
+        clock.tap do |c|
+          c.clock_name.should == 'Clock'
+          c.tick_interval.should == 1
+        end
+      end
+
+      it 'supports name overrides' do
+        clock('name' => 'testname').tap do |c|
+          c.clock_name.should == 'testname'
+        end
+      end
+
+      it 'supports interval overrides for floats' do
+        clock('tick_interval' => 1.5).tap do |c|
+          c.tick_interval.should == 1.5
+        end
+      end
+
+      it 'supports interval overrides for strings' do
+        clock('tick_interval' => '1.5').tap do |c|
+          c.tick_interval.should == 1.5
+        end
+      end
+
+      it 'should register a timer' do
+        EventMachine::PeriodicTimer.should_receive(:new).with(1)
+        clock.run!
+      end
+
+      it 'should generate a tick message when asked' do
+        clock.tap do |c|
+          now = Integer(Time.now.to_f * 1000)
+          messages.should be_empty
+          c.tick
+          messages.should have(1).message
+          messages.first.tap do |m|
+            m.data_type_name.should == 'RFlow::Message::Clock::Tick'
+            m.data.name.should == 'Clock'
+            m.data.timestamp.should >= now
+          end
+        end
+      end
+    end
+  end
+end

data/spec/rflow/configuration/ruby_dsl_spec.rb

@@ -0,0 +1,150 @@
+require 'spec_helper'
+require 'rflow/configuration'
+
+class RFlow
+  class Configuration
+    describe RubyDSL do
+      before(:each) do
+        ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+        Configuration.migrate_database
+      end
+
+      it "should correctly process an empty DSL" do
+        described_class.configure {}
+
+        Shard.should have(0).shards
+        Component.should have(0).components
+        Port.should have(0).ports
+        Connection.should have(0).connections
+      end
+
+      it "should correctly process a component declaration" do
+        described_class.configure do |c|
+          c.component 'boom', 'town', 'opt1' => 'OPT1', 'opt2' => 'OPT2'
+        end
+
+        Shard.should have(1).shard
+        Component.should have(1).component
+        Port.should have(0).ports
+        Connection.should have(0).connections
+
+        Component.first.tap do |c|
+          c.name.should == 'boom'
+          c.specification.should == 'town'
+          c.options.should == {'opt1' => 'OPT1', 'opt2' => 'OPT2'}
+        end
+      end
+
+      it "should correctly process a connect declaration" do
+        described_class.configure do |c|
+          c.component 'first', 'First'
+          c.component 'second', 'Second'
+          c.connect 'first#out' => 'second#in'
+          c.connect 'first#out' => 'second#in[inkey]'
+          c.connect 'first#out[outkey]' => 'second#in'
+          c.connect 'first#out[outkey]' => 'second#in[inkey]'
+        end
+
+        Shard.should have(1).shard
+        Component.should have(2).components
+        Port.should have(2).ports
+        Connection.should have(4).connections
+
+        first_component = Component.where(name: 'first').first.tap do |component|
+          component.specification.should == 'First'
+          component.should have(0).input_ports
+          component.should have(1).output_port
+          component.output_ports.first.name.should == 'out'
+
+          component.output_ports.first.should have(4).connections
+          component.output_ports.first.connections.tap do |connections|
+            connections[0].input_port_key.should be_nil
+            connections[0].output_port_key.should be_nil
+            connections[1].input_port_key.should == 'inkey'
+            connections[1].output_port_key.should be_nil
+            connections[2].input_port_key.should be_nil
+            connections[2].output_port_key.should == 'outkey'
+            connections[3].input_port_key.should == 'inkey'
+            connections[3].output_port_key.should == 'outkey'
+          end
+        end
+
+        Component.where(name: 'second').first.tap do |component|
+          component.specification.should == 'Second'
+          component.should have(1).input_port
+          component.input_ports.first.name.should == 'in'
+          component.should have(0).output_ports
+
+          component.input_ports.first.should have(4).connections
+          component.input_ports.first.connections.should == first_component.output_ports.first.connections
+        end
+      end
+
+      it "should correctly process shard declarations" do
+        described_class.configure do |c|
+          c.component 'first', 'First', :opt1 => 'opt1'
+
+          c.shard "s1", :process => 2 do |s|
+            s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
+          end
+
+          c.shard "s2", :type => :process, :count => 10 do |s|
+            s.component 'third', 'Third'
+            s.component 'fourth', 'Fourth'
+          end
+
+          c.shard "s-ignored", :type => :process, :count => 10 do
+            # ignored because there are no components
+          end
+
+          c.component 'fifth', 'Fifth'
+
+          c.connect 'first#out' => 'second#in'
+          c.connect 'second#out[outkey]' => 'third#in[inkey]'
+          c.connect 'second#out' => 'third#in2'
+          c.connect 'third#out' => 'fourth#in'
+          c.connect 'third#out' => 'fifth#in'
+        end
+
+        Shard.should have(3).shards
+        Component.should have(5).components
+        Port.should have(8).ports
+        Connection.should have(5).connections
+
+        Shard.all.tap do |shards|
+          shards.map(&:name).should == ['DEFAULT', 's1', 's2']
+          shards.first.components.all.map(&:name).should == ['first', 'fifth']
+          shards.second.components.all.map(&:name).should == ['second']
+          shards.third.components.all.map(&:name).should == ['third', 'fourth']
+        end
+
+        Port.all.map(&:name).should == ['out', 'in', 'out', 'in', 'in2', 'out', 'in', 'in']
+
+        Connection.all.map(&:name).should ==
+          ['first#out=>second#in',
+           'second#out[outkey]=>third#in[inkey]',
+           'second#out=>third#in2',
+           'third#out=>fourth#in',
+           'third#out=>fifth#in']
+      end
+
+      it "should not allow two components with the same name" do
+        expect {
+          described_class.configure do |c|
+            c.component 'first', 'First'
+            c.component 'first', 'First'
+          end
+        }.to raise_error(ActiveRecord::RecordInvalid)
+      end
+
+      it "should not allow two shards with the same name" do
+        expect {
+          described_class.configure do |c|
+            c.shard("s1", :process => 2) {|c| c.component 'x', 'y' }
+            c.shard("s1", :process => 2) {|c| c.component 'z', 'q' }
+          end
+        }.to raise_error
+      end
+    end
+  end
+end

data/spec/rflow/configuration_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'rflow/configuration'
+
+class RFlow
+  describe Configuration do
+    describe '.add_available_data_type' do
+      context 'if passed a data_serialization that is not avro' do
+        it "should throw an exception" do
+          expect { Configuration.add_available_data_type('A', 'boom', 'schema') }.to raise_error(
+            ArgumentError, "Data serialization_type must be 'avro' for 'A'")
+        end
+
+        it "should not update the available_data_types" do
+          expect {
+            Configuration.add_available_data_type('A', 'boom', 'schema') rescue nil
+          }.not_to change { Configuration.available_data_types.size }
+        end
+      end
+    end
+
+    describe "Data Extensions" do
+      describe ".add_available_data_extension" do
+        context 'if passed a non-module data extension' do
+          it "should throw an exception" do
+            expect {
+              Configuration.add_available_data_extension('data_type', 'NOTAMODULE')
+            }.to raise_error(ArgumentError, "Invalid data extension NOTAMODULE for data_type.  Only Ruby Modules allowed")
+          end
+        end
+
+        context "if passed a valid Module as a data extension" do
+          it "should update the available_data_extensions" do
+            expect {
+              Configuration.add_available_data_extension('data_type', Module.new)
+            }.to change { Configuration.available_data_extensions['data_type'].size }.by(1)
+          end
+        end
+      end
+
+      it "should perform simple 'prefix'-based inheritance for extensions" do
+        Configuration.add_available_data_extension('A', A = Module.new)
+        Configuration.add_available_data_extension('A::B', B = Module.new)
+        Configuration.add_available_data_extension('A::B::C', C = Module.new)
+        Configuration.add_available_data_extension('A::B::C::D', D = Module.new)
+
+        Configuration.available_data_extensions['A'].should == [A]
+        Configuration.available_data_extensions['A::B'].should == [A, B]
+        Configuration.available_data_extensions['A::B::C'].should == [A, B, C]
+        Configuration.available_data_extensions['A::B::C::D'].should == [A, B, C, D]
+        Configuration.available_data_extensions['A::B::C::D::E'].should == [A, B, C, D]
+      end
+    end
+  end
+end

data/spec/rflow/forward_to_input_port_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+class RFlow
+  describe ForwardToInputPort do
+    before(:each) do
+      ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+      Configuration.migrate_database
+    end
+
+    let(:filtered_message_connection) { RFlow::MessageCollectingConnection.new }
+    let(:dropped_message_connection) { RFlow::MessageCollectingConnection.new }
+
+    let(:generator) do
+      config = RFlow::Configuration::Component.new.tap do |c|
+        c.output_ports << RFlow::Configuration::OutputPort.new(name: 'out')
+      end
+      RFlow::Components::GenerateIntegerSequence.new(config).tap do |c|
+        c.configure! config.options
+        c.out.add_connection nil, ForwardToInputPort.new(ruby_proc_filter, 'in', nil)
+      end
+    end
+
+    let(:ruby_proc_filter) do
+      config = RFlow::Configuration::Component.new.tap do |c|
+        c.input_ports << RFlow::Configuration::InputPort.new(name: 'in')
+        ['filtered', 'dropped'].each {|p| c.output_ports << RFlow::Configuration::OutputPort.new(name: p) }
+        c.options = {'filter_proc_string' => 'message.data.data_object % 2 == 0'}
+      end
+      RFlow::Components::RubyProcFilter.new(config).tap do |c|
+        c.configure! config.options
+        c.filtered.add_connection nil, filtered_message_connection
+        c.dropped.add_connection nil, dropped_message_connection
+      end
+    end
+
+    def filtered_messages; filtered_message_connection.messages; end
+    def dropped_messages; dropped_message_connection.messages; end
+
+    it 'foo' do
+      5.times { generator.generate }
+      filtered_messages.should have(3).messages
+      filtered_messages.map(&:data).map(&:data_object).should == [0, 2, 4]
+      dropped_messages.should have(2).messages
+      dropped_messages.map(&:data).map(&:data_object).should == [1, 3]
+    end
+  end
+end
+

data/spec/rflow/forward_to_output_port_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+class RFlow
+  describe ForwardToOutputPort do
+    before(:each) do
+      ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+      Configuration.migrate_database
+    end
+
+    let(:message_connection) { RFlow::MessageCollectingConnection.new }
+
+    let(:generator) do
+      config = RFlow::Configuration::Component.new.tap do |c|
+        c.output_ports << RFlow::Configuration::OutputPort.new(name: 'out')
+      end
+      RFlow::Components::GenerateIntegerSequence.new(config).tap do |c|
+        c.configure! config.options
+        c.out.add_connection nil, ForwardToOutputPort.new(ruby_proc_filter, 'filtered')
+      end
+    end
+
+    let(:ruby_proc_filter) do
+      config = RFlow::Configuration::Component.new.tap do |c|
+        c.output_ports << RFlow::Configuration::OutputPort.new(name: 'filtered')
+        c.options = {'filter_proc_string' => 'message % 2 == 0'}
+      end
+      RFlow::Components::RubyProcFilter.new(config).tap do |c|
+        c.configure! config.options
+        c.filtered.add_connection nil, message_connection
+      end
+    end
+
+    def messages; message_connection.messages; end
+
+    it 'should place the messages on the output port, regardless of the filter' do
+      5.times { generator.generate }
+      messages.map(&:data).map(&:data_object).should == [0, 1, 2, 3, 4]
+    end
+  end
+end

data/spec/rflow/logger_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'open3'
+require 'rflow'
+
+class RFlow
+  describe Logger do
+    let(:log_file_path) { File.join(@temp_directory_path, 'logfile') }
+    let(:logger_config) do
+      {'rflow.log_file_path' => log_file_path,
+       'rflow.log_level' => 'DEBUG'}
+    end
+
+    def initialize_logger
+      @logger = described_class.new(logger_config)
+    end
+    let(:logger) { @logger }
+
+    before(:each) { initialize_logger }
+
+    it "should initialize correctly" do
+      File.exist?(log_file_path).should be true
+
+      logger.error "TESTTESTTEST"
+      File.read(log_file_path).should match(/TESTTESTTEST/)
+
+      logger.close
+    end
+
+    it "should reopen correctly" do
+      moved_path = log_file_path + '.old'
+
+      File.exist?(log_file_path).should be true
+      File.exist?(moved_path).should be false
+
+      File.rename log_file_path, moved_path
+
+      logger.reopen
+
+      logger.error "TESTTESTTEST"
+      File.read(log_file_path).should match(/TESTTESTTEST/)
+      File.read(moved_path).should_not match(/TESTTESTTEST/)
+
+      logger.close
+    end
+
+    it "should toggle log level"
+  end
+end

data/spec/rflow/message/data/raw_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'rflow/components/raw'
+
+class RFlow
+  class Message
+    class Data
+      describe 'Raw Avro Schema' do
+        let(:schema) { Configuration.available_data_types['RFlow::Message::Data::Raw']['avro'] }
+
+        it "should load the schema" do
+          schema.should_not be_nil
+        end
+
+        it "should encode and decode an object" do
+          raw = {'raw' => Array.new(256) { rand(256) }.pack('c*')}
+
+          expect { encode_avro(schema, raw) }.to_not raise_error
+          encoded = encode_avro(schema, raw)
+
+          expect { decode_avro(schema, encoded) }.to_not raise_error
+          decoded = decode_avro(schema, encoded)
+
+          decoded.should == raw
+          decoded['raw'].should == raw['raw']
+        end
+      end
+    end
+  end
+end