rflow 1.0.0a1 → 1.0.0a2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/Gemfile +0 -1
  5. data/NOTES +0 -13
  6. data/README.md +6 -1
  7. data/bin/rflow +2 -9
  8. data/example/basic_config.rb +1 -33
  9. data/example/basic_extensions.rb +0 -98
  10. data/example/http_config.rb +2 -3
  11. data/example/http_extensions.rb +6 -63
  12. data/lib/rflow.rb +31 -39
  13. data/lib/rflow/child_process.rb +112 -0
  14. data/lib/rflow/component.rb +77 -148
  15. data/lib/rflow/component/port.rb +38 -41
  16. data/lib/rflow/components.rb +4 -8
  17. data/lib/rflow/components/clock.rb +49 -0
  18. data/lib/rflow/components/integer.rb +39 -0
  19. data/lib/rflow/components/raw.rb +10 -6
  20. data/lib/rflow/components/replicate.rb +20 -0
  21. data/lib/rflow/components/ruby_proc_filter.rb +27 -0
  22. data/lib/rflow/configuration.rb +105 -184
  23. data/lib/rflow/configuration/component.rb +1 -4
  24. data/lib/rflow/configuration/connection.rb +11 -16
  25. data/lib/rflow/configuration/port.rb +3 -5
  26. data/lib/rflow/configuration/ruby_dsl.rb +105 -119
  27. data/lib/rflow/configuration/setting.rb +19 -25
  28. data/lib/rflow/configuration/shard.rb +1 -3
  29. data/lib/rflow/connection.rb +47 -10
  30. data/lib/rflow/connections.rb +0 -1
  31. data/lib/rflow/connections/zmq_connection.rb +34 -38
  32. data/lib/rflow/daemon_process.rb +155 -0
  33. data/lib/rflow/logger.rb +41 -25
  34. data/lib/rflow/master.rb +23 -105
  35. data/lib/rflow/message.rb +78 -108
  36. data/lib/rflow/pid_file.rb +37 -37
  37. data/lib/rflow/shard.rb +33 -100
  38. data/lib/rflow/version.rb +2 -2
  39. data/rflow.gemspec +2 -2
  40. data/schema/tick.avsc +10 -0
  41. data/spec/fixtures/config_ints.rb +4 -40
  42. data/spec/fixtures/config_shards.rb +1 -2
  43. data/spec/fixtures/extensions_ints.rb +0 -98
  44. data/spec/rflow/component/port_spec.rb +61 -0
  45. data/spec/rflow/components/clock_spec.rb +72 -0
  46. data/spec/rflow/configuration/ruby_dsl_spec.rb +150 -0
  47. data/spec/rflow/configuration_spec.rb +54 -0
  48. data/spec/rflow/forward_to_input_port_spec.rb +48 -0
  49. data/spec/rflow/forward_to_output_port_spec.rb +40 -0
  50. data/spec/rflow/logger_spec.rb +48 -0
  51. data/spec/rflow/message/data/raw_spec.rb +29 -0
  52. data/spec/rflow/message/data_spec.rb +58 -0
  53. data/spec/rflow/message_spec.rb +154 -0
  54. data/spec/rflow_spec.rb +94 -124
  55. data/spec/spec_helper.rb +8 -12
  56. metadata +46 -22
  57. data/lib/rflow/components/raw/extensions.rb +0 -18
  58. data/lib/rflow/port.rb +0 -4
  59. data/lib/rflow/util.rb +0 -19
  60. data/spec/rflow_component_port_spec.rb +0 -58
  61. data/spec/rflow_configuration_ruby_dsl_spec.rb +0 -148
  62. data/spec/rflow_configuration_spec.rb +0 -73
  63. data/spec/rflow_message_data_raw.rb +0 -26
  64. data/spec/rflow_message_data_spec.rb +0 -60
  65. data/spec/rflow_message_spec.rb +0 -182
  66. data/spec/schema_spec.rb +0 -28
  67. data/temp.rb +0 -295
@@ -0,0 +1,72 @@
1
+ require 'spec_helper'
2
+
3
+ class RFlow
4
+ module Components
5
+ describe Clock do
6
+ before(:each) do
7
+ ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
8
+ Configuration.migrate_database
9
+ end
10
+ let(:config) do
11
+ RFlow::Configuration::Component.new.tap do |c|
12
+ c.output_ports << RFlow::Configuration::OutputPort.new(name: 'tick_port')
13
+ end
14
+ end
15
+ let(:message_connection) { RFlow::MessageCollectingConnection.new }
16
+
17
+ def clock(args = {})
18
+ Clock.new(config).tap do |c|
19
+ c.configure! args
20
+ c.tick_port.connect!
21
+ c.tick_port.add_connection nil, message_connection
22
+ end
23
+ end
24
+
25
+ def messages; message_connection.messages; end
26
+
27
+ it 'defaults configuration nicely' do
28
+ clock.tap do |c|
29
+ c.clock_name.should == 'Clock'
30
+ c.tick_interval.should == 1
31
+ end
32
+ end
33
+
34
+ it 'supports name overrides' do
35
+ clock('name' => 'testname').tap do |c|
36
+ c.clock_name.should == 'testname'
37
+ end
38
+ end
39
+
40
+ it 'supports interval overrides for floats' do
41
+ clock('tick_interval' => 1.5).tap do |c|
42
+ c.tick_interval.should == 1.5
43
+ end
44
+ end
45
+
46
+ it 'supports interval overrides for strings' do
47
+ clock('tick_interval' => '1.5').tap do |c|
48
+ c.tick_interval.should == 1.5
49
+ end
50
+ end
51
+
52
+ it 'should register a timer' do
53
+ EventMachine::PeriodicTimer.should_receive(:new).with(1)
54
+ clock.run!
55
+ end
56
+
57
+ it 'should generate a tick message when asked' do
58
+ clock.tap do |c|
59
+ now = Integer(Time.now.to_f * 1000)
60
+ messages.should be_empty
61
+ c.tick
62
+ messages.should have(1).message
63
+ messages.first.tap do |m|
64
+ m.data_type_name.should == 'RFlow::Message::Clock::Tick'
65
+ m.data.name.should == 'Clock'
66
+ m.data.timestamp.should >= now
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,150 @@
1
+ require 'spec_helper'
2
+ require 'rflow/configuration'
3
+
4
+ class RFlow
5
+ class Configuration
6
+ describe RubyDSL do
7
+ before(:each) do
8
+ ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
9
+ Configuration.migrate_database
10
+ end
11
+
12
+ it "should correctly process an empty DSL" do
13
+ described_class.configure {}
14
+
15
+ Shard.should have(0).shards
16
+ Component.should have(0).components
17
+ Port.should have(0).ports
18
+ Connection.should have(0).connections
19
+ end
20
+
21
+ it "should correctly process a component declaration" do
22
+ described_class.configure do |c|
23
+ c.component 'boom', 'town', 'opt1' => 'OPT1', 'opt2' => 'OPT2'
24
+ end
25
+
26
+ Shard.should have(1).shard
27
+ Component.should have(1).component
28
+ Port.should have(0).ports
29
+ Connection.should have(0).connections
30
+
31
+ Component.first.tap do |c|
32
+ c.name.should == 'boom'
33
+ c.specification.should == 'town'
34
+ c.options.should == {'opt1' => 'OPT1', 'opt2' => 'OPT2'}
35
+ end
36
+ end
37
+
38
+ it "should correctly process a connect declaration" do
39
+ described_class.configure do |c|
40
+ c.component 'first', 'First'
41
+ c.component 'second', 'Second'
42
+ c.connect 'first#out' => 'second#in'
43
+ c.connect 'first#out' => 'second#in[inkey]'
44
+ c.connect 'first#out[outkey]' => 'second#in'
45
+ c.connect 'first#out[outkey]' => 'second#in[inkey]'
46
+ end
47
+
48
+ Shard.should have(1).shard
49
+ Component.should have(2).components
50
+ Port.should have(2).ports
51
+ Connection.should have(4).connections
52
+
53
+ first_component = Component.where(name: 'first').first.tap do |component|
54
+ component.specification.should == 'First'
55
+ component.should have(0).input_ports
56
+ component.should have(1).output_port
57
+ component.output_ports.first.name.should == 'out'
58
+
59
+ component.output_ports.first.should have(4).connections
60
+ component.output_ports.first.connections.tap do |connections|
61
+ connections[0].input_port_key.should be_nil
62
+ connections[0].output_port_key.should be_nil
63
+ connections[1].input_port_key.should == 'inkey'
64
+ connections[1].output_port_key.should be_nil
65
+ connections[2].input_port_key.should be_nil
66
+ connections[2].output_port_key.should == 'outkey'
67
+ connections[3].input_port_key.should == 'inkey'
68
+ connections[3].output_port_key.should == 'outkey'
69
+ end
70
+ end
71
+
72
+ Component.where(name: 'second').first.tap do |component|
73
+ component.specification.should == 'Second'
74
+ component.should have(1).input_port
75
+ component.input_ports.first.name.should == 'in'
76
+ component.should have(0).output_ports
77
+
78
+ component.input_ports.first.should have(4).connections
79
+ component.input_ports.first.connections.should == first_component.output_ports.first.connections
80
+ end
81
+ end
82
+
83
+ it "should correctly process shard declarations" do
84
+ described_class.configure do |c|
85
+ c.component 'first', 'First', :opt1 => 'opt1'
86
+
87
+ c.shard "s1", :process => 2 do |s|
88
+ s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
89
+ end
90
+
91
+ c.shard "s2", :type => :process, :count => 10 do |s|
92
+ s.component 'third', 'Third'
93
+ s.component 'fourth', 'Fourth'
94
+ end
95
+
96
+ c.shard "s-ignored", :type => :process, :count => 10 do
97
+ # ignored because there are no components
98
+ end
99
+
100
+ c.component 'fifth', 'Fifth'
101
+
102
+ c.connect 'first#out' => 'second#in'
103
+ c.connect 'second#out[outkey]' => 'third#in[inkey]'
104
+ c.connect 'second#out' => 'third#in2'
105
+ c.connect 'third#out' => 'fourth#in'
106
+ c.connect 'third#out' => 'fifth#in'
107
+ end
108
+
109
+ Shard.should have(3).shards
110
+ Component.should have(5).components
111
+ Port.should have(8).ports
112
+ Connection.should have(5).connections
113
+
114
+ Shard.all.tap do |shards|
115
+ shards.map(&:name).should == ['DEFAULT', 's1', 's2']
116
+ shards.first.components.all.map(&:name).should == ['first', 'fifth']
117
+ shards.second.components.all.map(&:name).should == ['second']
118
+ shards.third.components.all.map(&:name).should == ['third', 'fourth']
119
+ end
120
+
121
+ Port.all.map(&:name).should == ['out', 'in', 'out', 'in', 'in2', 'out', 'in', 'in']
122
+
123
+ Connection.all.map(&:name).should ==
124
+ ['first#out=>second#in',
125
+ 'second#out[outkey]=>third#in[inkey]',
126
+ 'second#out=>third#in2',
127
+ 'third#out=>fourth#in',
128
+ 'third#out=>fifth#in']
129
+ end
130
+
131
+ it "should not allow two components with the same name" do
132
+ expect {
133
+ described_class.configure do |c|
134
+ c.component 'first', 'First'
135
+ c.component 'first', 'First'
136
+ end
137
+ }.to raise_error(ActiveRecord::RecordInvalid)
138
+ end
139
+
140
+ it "should not allow two shards with the same name" do
141
+ expect {
142
+ described_class.configure do |c|
143
+ c.shard("s1", :process => 2) {|c| c.component 'x', 'y' }
144
+ c.shard("s1", :process => 2) {|c| c.component 'z', 'q' }
145
+ end
146
+ }.to raise_error
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+ require 'rflow/configuration'
3
+
4
+ class RFlow
5
+ describe Configuration do
6
+ describe '.add_available_data_type' do
7
+ context 'if passed a data_serialization that is not avro' do
8
+ it "should throw an exception" do
9
+ expect { Configuration.add_available_data_type('A', 'boom', 'schema') }.to raise_error(
10
+ ArgumentError, "Data serialization_type must be 'avro' for 'A'")
11
+ end
12
+
13
+ it "should not update the available_data_types" do
14
+ expect {
15
+ Configuration.add_available_data_type('A', 'boom', 'schema') rescue nil
16
+ }.not_to change { Configuration.available_data_types.size }
17
+ end
18
+ end
19
+ end
20
+
21
+ describe "Data Extensions" do
22
+ describe ".add_available_data_extension" do
23
+ context 'if passed a non-module data extension' do
24
+ it "should throw an exception" do
25
+ expect {
26
+ Configuration.add_available_data_extension('data_type', 'NOTAMODULE')
27
+ }.to raise_error(ArgumentError, "Invalid data extension NOTAMODULE for data_type. Only Ruby Modules allowed")
28
+ end
29
+ end
30
+
31
+ context "if passed a valid Module as a data extension" do
32
+ it "should update the available_data_extensions" do
33
+ expect {
34
+ Configuration.add_available_data_extension('data_type', Module.new)
35
+ }.to change { Configuration.available_data_extensions['data_type'].size }.by(1)
36
+ end
37
+ end
38
+ end
39
+
40
+ it "should perform simple 'prefix'-based inheritance for extensions" do
41
+ Configuration.add_available_data_extension('A', A = Module.new)
42
+ Configuration.add_available_data_extension('A::B', B = Module.new)
43
+ Configuration.add_available_data_extension('A::B::C', C = Module.new)
44
+ Configuration.add_available_data_extension('A::B::C::D', D = Module.new)
45
+
46
+ Configuration.available_data_extensions['A'].should == [A]
47
+ Configuration.available_data_extensions['A::B'].should == [A, B]
48
+ Configuration.available_data_extensions['A::B::C'].should == [A, B, C]
49
+ Configuration.available_data_extensions['A::B::C::D'].should == [A, B, C, D]
50
+ Configuration.available_data_extensions['A::B::C::D::E'].should == [A, B, C, D]
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,48 @@
1
+ require 'spec_helper'
2
+
3
+ class RFlow
4
+ describe ForwardToInputPort do
5
+ before(:each) do
6
+ ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
7
+ Configuration.migrate_database
8
+ end
9
+
10
+ let(:filtered_message_connection) { RFlow::MessageCollectingConnection.new }
11
+ let(:dropped_message_connection) { RFlow::MessageCollectingConnection.new }
12
+
13
+ let(:generator) do
14
+ config = RFlow::Configuration::Component.new.tap do |c|
15
+ c.output_ports << RFlow::Configuration::OutputPort.new(name: 'out')
16
+ end
17
+ RFlow::Components::GenerateIntegerSequence.new(config).tap do |c|
18
+ c.configure! config.options
19
+ c.out.add_connection nil, ForwardToInputPort.new(ruby_proc_filter, 'in', nil)
20
+ end
21
+ end
22
+
23
+ let(:ruby_proc_filter) do
24
+ config = RFlow::Configuration::Component.new.tap do |c|
25
+ c.input_ports << RFlow::Configuration::InputPort.new(name: 'in')
26
+ ['filtered', 'dropped'].each {|p| c.output_ports << RFlow::Configuration::OutputPort.new(name: p) }
27
+ c.options = {'filter_proc_string' => 'message.data.data_object % 2 == 0'}
28
+ end
29
+ RFlow::Components::RubyProcFilter.new(config).tap do |c|
30
+ c.configure! config.options
31
+ c.filtered.add_connection nil, filtered_message_connection
32
+ c.dropped.add_connection nil, dropped_message_connection
33
+ end
34
+ end
35
+
36
+ def filtered_messages; filtered_message_connection.messages; end
37
+ def dropped_messages; dropped_message_connection.messages; end
38
+
39
+ it 'foo' do
40
+ 5.times { generator.generate }
41
+ filtered_messages.should have(3).messages
42
+ filtered_messages.map(&:data).map(&:data_object).should == [0, 2, 4]
43
+ dropped_messages.should have(2).messages
44
+ dropped_messages.map(&:data).map(&:data_object).should == [1, 3]
45
+ end
46
+ end
47
+ end
48
+
@@ -0,0 +1,40 @@
1
+ require 'spec_helper'
2
+
3
+ class RFlow
4
+ describe ForwardToOutputPort do
5
+ before(:each) do
6
+ ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
7
+ Configuration.migrate_database
8
+ end
9
+
10
+ let(:message_connection) { RFlow::MessageCollectingConnection.new }
11
+
12
+ let(:generator) do
13
+ config = RFlow::Configuration::Component.new.tap do |c|
14
+ c.output_ports << RFlow::Configuration::OutputPort.new(name: 'out')
15
+ end
16
+ RFlow::Components::GenerateIntegerSequence.new(config).tap do |c|
17
+ c.configure! config.options
18
+ c.out.add_connection nil, ForwardToOutputPort.new(ruby_proc_filter, 'filtered')
19
+ end
20
+ end
21
+
22
+ let(:ruby_proc_filter) do
23
+ config = RFlow::Configuration::Component.new.tap do |c|
24
+ c.output_ports << RFlow::Configuration::OutputPort.new(name: 'filtered')
25
+ c.options = {'filter_proc_string' => 'message % 2 == 0'}
26
+ end
27
+ RFlow::Components::RubyProcFilter.new(config).tap do |c|
28
+ c.configure! config.options
29
+ c.filtered.add_connection nil, message_connection
30
+ end
31
+ end
32
+
33
+ def messages; message_connection.messages; end
34
+
35
+ it 'should place the messages on the output port, regardless of the filter' do
36
+ 5.times { generator.generate }
37
+ messages.map(&:data).map(&:data_object).should == [0, 1, 2, 3, 4]
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,48 @@
1
+ require 'spec_helper'
2
+ require 'open3'
3
+ require 'rflow'
4
+
5
+ class RFlow
6
+ describe Logger do
7
+ let(:log_file_path) { File.join(@temp_directory_path, 'logfile') }
8
+ let(:logger_config) do
9
+ {'rflow.log_file_path' => log_file_path,
10
+ 'rflow.log_level' => 'DEBUG'}
11
+ end
12
+
13
+ def initialize_logger
14
+ @logger = described_class.new(logger_config)
15
+ end
16
+ let(:logger) { @logger }
17
+
18
+ before(:each) { initialize_logger }
19
+
20
+ it "should initialize correctly" do
21
+ File.exist?(log_file_path).should be true
22
+
23
+ logger.error "TESTTESTTEST"
24
+ File.read(log_file_path).should match(/TESTTESTTEST/)
25
+
26
+ logger.close
27
+ end
28
+
29
+ it "should reopen correctly" do
30
+ moved_path = log_file_path + '.old'
31
+
32
+ File.exist?(log_file_path).should be true
33
+ File.exist?(moved_path).should be false
34
+
35
+ File.rename log_file_path, moved_path
36
+
37
+ logger.reopen
38
+
39
+ logger.error "TESTTESTTEST"
40
+ File.read(log_file_path).should match(/TESTTESTTEST/)
41
+ File.read(moved_path).should_not match(/TESTTESTTEST/)
42
+
43
+ logger.close
44
+ end
45
+
46
+ it "should toggle log level"
47
+ end
48
+ end
@@ -0,0 +1,29 @@
1
+ require 'spec_helper'
2
+ require 'rflow/components/raw'
3
+
4
+ class RFlow
5
+ class Message
6
+ class Data
7
+ describe 'Raw Avro Schema' do
8
+ let(:schema) { Configuration.available_data_types['RFlow::Message::Data::Raw']['avro'] }
9
+
10
+ it "should load the schema" do
11
+ schema.should_not be_nil
12
+ end
13
+
14
+ it "should encode and decode an object" do
15
+ raw = {'raw' => Array.new(256) { rand(256) }.pack('c*')}
16
+
17
+ expect { encode_avro(schema, raw) }.to_not raise_error
18
+ encoded = encode_avro(schema, raw)
19
+
20
+ expect { decode_avro(schema, encoded) }.to_not raise_error
21
+ decoded = decode_avro(schema, encoded)
22
+
23
+ decoded.should == raw
24
+ decoded['raw'].should == raw['raw']
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end