rflow 1.0.0a1 → 1.0.0a2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/Gemfile +0 -1
  5. data/NOTES +0 -13
  6. data/README.md +6 -1
  7. data/bin/rflow +2 -9
  8. data/example/basic_config.rb +1 -33
  9. data/example/basic_extensions.rb +0 -98
  10. data/example/http_config.rb +2 -3
  11. data/example/http_extensions.rb +6 -63
  12. data/lib/rflow.rb +31 -39
  13. data/lib/rflow/child_process.rb +112 -0
  14. data/lib/rflow/component.rb +77 -148
  15. data/lib/rflow/component/port.rb +38 -41
  16. data/lib/rflow/components.rb +4 -8
  17. data/lib/rflow/components/clock.rb +49 -0
  18. data/lib/rflow/components/integer.rb +39 -0
  19. data/lib/rflow/components/raw.rb +10 -6
  20. data/lib/rflow/components/replicate.rb +20 -0
  21. data/lib/rflow/components/ruby_proc_filter.rb +27 -0
  22. data/lib/rflow/configuration.rb +105 -184
  23. data/lib/rflow/configuration/component.rb +1 -4
  24. data/lib/rflow/configuration/connection.rb +11 -16
  25. data/lib/rflow/configuration/port.rb +3 -5
  26. data/lib/rflow/configuration/ruby_dsl.rb +105 -119
  27. data/lib/rflow/configuration/setting.rb +19 -25
  28. data/lib/rflow/configuration/shard.rb +1 -3
  29. data/lib/rflow/connection.rb +47 -10
  30. data/lib/rflow/connections.rb +0 -1
  31. data/lib/rflow/connections/zmq_connection.rb +34 -38
  32. data/lib/rflow/daemon_process.rb +155 -0
  33. data/lib/rflow/logger.rb +41 -25
  34. data/lib/rflow/master.rb +23 -105
  35. data/lib/rflow/message.rb +78 -108
  36. data/lib/rflow/pid_file.rb +37 -37
  37. data/lib/rflow/shard.rb +33 -100
  38. data/lib/rflow/version.rb +2 -2
  39. data/rflow.gemspec +2 -2
  40. data/schema/tick.avsc +10 -0
  41. data/spec/fixtures/config_ints.rb +4 -40
  42. data/spec/fixtures/config_shards.rb +1 -2
  43. data/spec/fixtures/extensions_ints.rb +0 -98
  44. data/spec/rflow/component/port_spec.rb +61 -0
  45. data/spec/rflow/components/clock_spec.rb +72 -0
  46. data/spec/rflow/configuration/ruby_dsl_spec.rb +150 -0
  47. data/spec/rflow/configuration_spec.rb +54 -0
  48. data/spec/rflow/forward_to_input_port_spec.rb +48 -0
  49. data/spec/rflow/forward_to_output_port_spec.rb +40 -0
  50. data/spec/rflow/logger_spec.rb +48 -0
  51. data/spec/rflow/message/data/raw_spec.rb +29 -0
  52. data/spec/rflow/message/data_spec.rb +58 -0
  53. data/spec/rflow/message_spec.rb +154 -0
  54. data/spec/rflow_spec.rb +94 -124
  55. data/spec/spec_helper.rb +8 -12
  56. metadata +46 -22
  57. data/lib/rflow/components/raw/extensions.rb +0 -18
  58. data/lib/rflow/port.rb +0 -4
  59. data/lib/rflow/util.rb +0 -19
  60. data/spec/rflow_component_port_spec.rb +0 -58
  61. data/spec/rflow_configuration_ruby_dsl_spec.rb +0 -148
  62. data/spec/rflow_configuration_spec.rb +0 -73
  63. data/spec/rflow_message_data_raw.rb +0 -26
  64. data/spec/rflow_message_data_spec.rb +0 -60
  65. data/spec/rflow_message_spec.rb +0 -182
  66. data/spec/schema_spec.rb +0 -28
  67. data/temp.rb +0 -295
data/spec/spec_helper.rb CHANGED
@@ -2,11 +2,15 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'rflow')
2
2
 
3
3
  require 'fileutils'
4
4
  require 'log4r'
5
+ require 'rspec/collection_matchers'
6
+
7
+ I18n.enforce_available_locales = true
5
8
 
6
9
  RSpec.configure do |config|
7
10
  config.before(:all) do
8
11
  RFlow.logger = Log4r::Logger.new 'test'
9
12
  RFlow.logger.add Log4r::StdoutOutputter.new('test_stdout', :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
13
+ RFlow.logger.level = 5
10
14
  @base_temp_directory_path = File.join(File.dirname(__FILE__), 'tmp')
11
15
  end
12
16
 
@@ -21,19 +25,11 @@ RSpec.configure do |config|
21
25
  end
22
26
  end
23
27
 
24
-
25
- def decode_avro(schema_string, serialized_object)
28
+ def decode_avro(schema_string, bytes)
26
29
  schema = Avro::Schema.parse(schema_string)
27
- serialized_object.force_encoding 'BINARY'
28
- sio = StringIO.new(serialized_object)
29
- Avro::IO::DatumReader.new(schema, schema).read Avro::IO::BinaryDecoder.new(sio)
30
+ RFlow::Avro.decode(Avro::IO::DatumReader.new(schema, schema), bytes)
30
31
  end
31
32
 
32
- def encode_avro(schema_string, object)
33
- encoded_string = ''
34
- encoded_string.force_encoding 'BINARY'
35
- schema = Avro::Schema.parse(schema_string)
36
- sio = StringIO.new(encoded_string)
37
- Avro::IO::DatumWriter.new(schema).write object, Avro::IO::BinaryEncoder.new(sio)
38
- encoded_string
33
+ def encode_avro(schema_string, message)
34
+ RFlow::Avro.encode(Avro::IO::DatumWriter.new(Avro::Schema.parse(schema_string)), message)
39
35
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0a1
4
+ version: 1.0.0a2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael L. Artz
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-01 00:00:00.000000000 Z
11
+ date: 2014-06-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: uuidtools
@@ -114,14 +114,28 @@ dependencies:
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '2.6'
117
+ version: '2.99'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '2.6'
124
+ version: '2.99'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rspec-collection_matchers
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: 0.0.4
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: 0.0.4
125
139
  - !ruby/object:Gem::Dependency
126
140
  name: rake
127
141
  requirement: !ruby/object:Gem::Requirement
@@ -160,6 +174,7 @@ extensions: []
160
174
  extra_rdoc_files: []
161
175
  files:
162
176
  - ".gitignore"
177
+ - ".rspec"
163
178
  - ".ruby-gemset"
164
179
  - ".ruby-version"
165
180
  - ".travis.yml"
@@ -176,11 +191,15 @@ files:
176
191
  - example/http_config.rb
177
192
  - example/http_extensions.rb
178
193
  - lib/rflow.rb
194
+ - lib/rflow/child_process.rb
179
195
  - lib/rflow/component.rb
180
196
  - lib/rflow/component/port.rb
181
197
  - lib/rflow/components.rb
198
+ - lib/rflow/components/clock.rb
199
+ - lib/rflow/components/integer.rb
182
200
  - lib/rflow/components/raw.rb
183
- - lib/rflow/components/raw/extensions.rb
201
+ - lib/rflow/components/replicate.rb
202
+ - lib/rflow/components/ruby_proc_filter.rb
184
203
  - lib/rflow/configuration.rb
185
204
  - lib/rflow/configuration/component.rb
186
205
  - lib/rflow/configuration/connection.rb
@@ -197,30 +216,32 @@ files:
197
216
  - lib/rflow/connection.rb
198
217
  - lib/rflow/connections.rb
199
218
  - lib/rflow/connections/zmq_connection.rb
219
+ - lib/rflow/daemon_process.rb
200
220
  - lib/rflow/logger.rb
201
221
  - lib/rflow/master.rb
202
222
  - lib/rflow/message.rb
203
223
  - lib/rflow/pid_file.rb
204
- - lib/rflow/port.rb
205
224
  - lib/rflow/shard.rb
206
- - lib/rflow/util.rb
207
225
  - lib/rflow/version.rb
208
226
  - rflow.gemspec
209
227
  - schema/message.avsc
210
228
  - schema/raw.avsc
229
+ - schema/tick.avsc
211
230
  - spec/fixtures/config_ints.rb
212
231
  - spec/fixtures/config_shards.rb
213
232
  - spec/fixtures/extensions_ints.rb
214
- - spec/rflow_component_port_spec.rb
215
- - spec/rflow_configuration_ruby_dsl_spec.rb
216
- - spec/rflow_configuration_spec.rb
217
- - spec/rflow_message_data_raw.rb
218
- - spec/rflow_message_data_spec.rb
219
- - spec/rflow_message_spec.rb
233
+ - spec/rflow/component/port_spec.rb
234
+ - spec/rflow/components/clock_spec.rb
235
+ - spec/rflow/configuration/ruby_dsl_spec.rb
236
+ - spec/rflow/configuration_spec.rb
237
+ - spec/rflow/forward_to_input_port_spec.rb
238
+ - spec/rflow/forward_to_output_port_spec.rb
239
+ - spec/rflow/logger_spec.rb
240
+ - spec/rflow/message/data/raw_spec.rb
241
+ - spec/rflow/message/data_spec.rb
242
+ - spec/rflow/message_spec.rb
220
243
  - spec/rflow_spec.rb
221
- - spec/schema_spec.rb
222
244
  - spec/spec_helper.rb
223
- - temp.rb
224
245
  homepage: https://github.com/redjack/rflow
225
246
  licenses:
226
247
  - Apache-2.0
@@ -249,13 +270,16 @@ test_files:
249
270
  - spec/fixtures/config_ints.rb
250
271
  - spec/fixtures/config_shards.rb
251
272
  - spec/fixtures/extensions_ints.rb
252
- - spec/rflow_component_port_spec.rb
253
- - spec/rflow_configuration_ruby_dsl_spec.rb
254
- - spec/rflow_configuration_spec.rb
255
- - spec/rflow_message_data_raw.rb
256
- - spec/rflow_message_data_spec.rb
257
- - spec/rflow_message_spec.rb
273
+ - spec/rflow/component/port_spec.rb
274
+ - spec/rflow/components/clock_spec.rb
275
+ - spec/rflow/configuration/ruby_dsl_spec.rb
276
+ - spec/rflow/configuration_spec.rb
277
+ - spec/rflow/forward_to_input_port_spec.rb
278
+ - spec/rflow/forward_to_output_port_spec.rb
279
+ - spec/rflow/logger_spec.rb
280
+ - spec/rflow/message/data/raw_spec.rb
281
+ - spec/rflow/message/data_spec.rb
282
+ - spec/rflow/message_spec.rb
258
283
  - spec/rflow_spec.rb
259
- - spec/schema_spec.rb
260
284
  - spec/spec_helper.rb
261
285
  has_rdoc:
@@ -1,18 +0,0 @@
1
- class RFlow
2
- module Components
3
- module Raw
4
-
5
- module Extensions
6
-
7
- module RawExtension
8
- def self.extended(base_data)
9
- base_data.data_object ||= {'raw' => ''}
10
- end
11
-
12
- def raw; data_object['raw']; end
13
- def raw=(new_raw); data_object['raw'] = new_raw; end
14
- end
15
- end
16
- end
17
- end
18
- end
data/lib/rflow/port.rb DELETED
@@ -1,4 +0,0 @@
1
- class RFlow
2
- class Port
3
- end # class Port
4
- end # class RFlow
data/lib/rflow/util.rb DELETED
@@ -1,19 +0,0 @@
1
- require 'uuidtools'
2
-
3
- class RFlow
4
- module Util
5
- # Generate a UUID based on either the SHA1 of a seed string (v5) with a
6
- # 'zero' UUID namespace, or using a purely random generation
7
- # (v4) if no seed string is present
8
- def generate_uuid_string(seed=nil)
9
- uuid = if seed
10
- UUIDTools::UUID.sha1_create(UUIDTools::UUID.parse_int(0), seed)
11
- else
12
- UUIDTools::UUID.random_create
13
- end
14
- uuid.to_s
15
- end
16
-
17
-
18
- end
19
- end
@@ -1,58 +0,0 @@
1
- require 'spec_helper.rb'
2
-
3
- describe RFlow::Component::Port do
4
- it "should not be connected" do
5
- described_class.new.connected?.should be_false
6
- end
7
- end
8
-
9
- describe RFlow::Component::HashPort do
10
- it "should not be connected" do
11
- port_config = double('Port Config')
12
- port_config.should_receive(:name).and_return('port')
13
- port_config.should_receive(:uuid).and_return('1')
14
-
15
- port = described_class.new(port_config)
16
- port.connected?.should be_false
17
- end
18
- end
19
-
20
- describe RFlow::Component::InputPort do
21
- context ".connect!" do
22
- it "should be connected" do
23
- connection_double = double('connection')
24
- connection_double.should_receive(:connect_input!)
25
-
26
- port_config = double('Port Config')
27
- port_config.should_receive(:name).and_return('port')
28
- port_config.should_receive(:uuid).and_return('1')
29
-
30
- port = described_class.new(port_config)
31
- port.add_connection(nil, connection_double)
32
-
33
- port.connected?.should be_false
34
- port.connect!
35
- port.connected?.should be_true
36
- end
37
- end
38
- end
39
-
40
- describe RFlow::Component::OutputPort do
41
- context ".connect!" do
42
- it "shouldbe connected" do
43
- connection_double = double('connection')
44
- connection_double.should_receive(:connect_output!)
45
-
46
- port_config = double('Port Config')
47
- port_config.should_receive(:name).and_return('port')
48
- port_config.should_receive(:uuid).and_return('1')
49
-
50
- port = described_class.new(port_config)
51
- port.add_connection(nil, connection_double)
52
-
53
- port.connected?.should be_false
54
- port.connect!
55
- port.connected?.should be_true
56
- end
57
- end
58
- end
@@ -1,148 +0,0 @@
1
- require 'spec_helper.rb'
2
- require 'rflow/configuration'
3
-
4
- describe RFlow::Configuration::RubyDSL do
5
- before(:each) do
6
- ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
7
- RFlow::Configuration.migrate_database
8
- end
9
-
10
- it "should correctly process an empty DSL" do
11
- described_class.configure { |c| }
12
-
13
- config = RFlow::Configuration.new
14
- RFlow::Configuration::Shard.count.should == 1
15
- RFlow::Configuration::Component.count.should == 0
16
- RFlow::Configuration::Port.count.should == 0
17
- RFlow::Configuration::Connection.count.should == 0
18
-
19
- puts config.to_s
20
- end
21
-
22
- it "should correctly process a component declaration" do
23
- described_class.configure do |c|
24
- c.component 'boom', 'town', 'opt1' => 'OPT1', 'opt2' => 'OPT2'
25
- end
26
-
27
- config = RFlow::Configuration.new
28
- RFlow::Configuration::Shard.count.should == 1
29
- RFlow::Configuration::Component.count.should == 1
30
- RFlow::Configuration::Port.count.should == 0
31
- RFlow::Configuration::Connection.count.should == 0
32
-
33
- component = RFlow::Configuration::Component.all.first
34
- component.name.should == 'boom'
35
- component.specification.should == 'town'
36
- component.options.should == {'opt1' => 'OPT1', 'opt2' => 'OPT2'}
37
- end
38
-
39
- it "should correctly process a connect declaration" do
40
- described_class.configure do |c|
41
- c.component 'first', 'First'
42
- c.component 'second', 'Second'
43
- c.connect 'first#out' => 'second#in'
44
- c.connect 'first#out' => 'second#in[inkey]'
45
- c.connect 'first#out[outkey]' => 'second#in'
46
- c.connect 'first#out[outkey]' => 'second#in[inkey]'
47
- end
48
-
49
- config = RFlow::Configuration.new
50
- RFlow::Configuration::Shard.count.should == 1
51
- RFlow::Configuration::Component.count.should == 2
52
- RFlow::Configuration::Port.count.should == 2
53
- RFlow::Configuration::Connection.count.should == 4
54
-
55
- first_component = RFlow::Configuration::Component.where(name: 'first').first
56
- second_component = RFlow::Configuration::Component.where(name: 'second').first
57
-
58
- first_component.specification.should == 'First'
59
- first_component.input_ports.count.should == 0
60
- first_component.output_ports.count.should == 1
61
- first_component.output_ports.first.name.should == 'out'
62
- first_connections = first_component.output_ports.first.connections.all
63
- first_connections.count.should == 4
64
- first_connections[0].input_port_key.should be_nil
65
- first_connections[0].output_port_key.should be_nil
66
- first_connections[1].input_port_key.should == 'inkey'
67
- first_connections[1].output_port_key.should be_nil
68
- first_connections[2].input_port_key.should be_nil
69
- first_connections[2].output_port_key.should == 'outkey'
70
- first_connections[3].input_port_key.should == 'inkey'
71
- first_connections[3].output_port_key.should == 'outkey'
72
-
73
- second_component.specification.should == 'Second'
74
- second_component.input_ports.count.should == 1
75
- second_component.output_ports.count.should == 0
76
- second_component.input_ports.first.name.should == 'in'
77
- second_connections = second_component.input_ports.first.connections.all
78
- second_connections.count.should == 4
79
-
80
- first_connections.should == second_connections
81
-
82
- puts config.to_s
83
- end
84
-
85
- it "should correctly process shard declarations" do
86
- described_class.configure do |c|
87
- c.component 'first', 'First', :opt1 => 'opt1'
88
-
89
- c.shard "s1", :process => 2 do |s|
90
- s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
91
- end
92
-
93
- c.shard "s2", :type => :process, :count => 10 do |s|
94
- s.component 'third', 'Third'
95
- s.component 'fourth', 'Fourth'
96
- end
97
-
98
- c.component 'fifth', 'Fifth'
99
-
100
- c.connect 'first#out' => 'second#in'
101
- c.connect 'second#out[outkey]' => 'third#in[inkey]'
102
- c.connect 'second#out' => 'third#in2'
103
- c.connect 'third#out' => 'fourth#in'
104
- c.connect 'third#out' => 'fifth#in'
105
- end
106
-
107
- config = RFlow::Configuration.new
108
- RFlow::Configuration::Shard.count.should == 3
109
- RFlow::Configuration::Component.count.should == 5
110
- RFlow::Configuration::Port.count.should == 8
111
- RFlow::Configuration::Connection.count.should == 5
112
-
113
- shards = RFlow::Configuration::Shard.all
114
- shards.map(&:name).should == ['DEFAULT', 's1', 's2']
115
- shards.first.components.all.map(&:name).should == ['first', 'fifth']
116
- shards.second.components.all.map(&:name).should == ['second']
117
- shards.third.components.all.map(&:name).should == ['third', 'fourth']
118
-
119
- RFlow::Configuration::Port.all.map(&:name).should == ['out', 'in', 'out', 'in', 'in2', 'out', 'in', 'in']
120
-
121
- RFlow::Configuration::Connection.all.map(&:name).should == ['first#out=>second#in',
122
- 'second#out[outkey]=>third#in[inkey]',
123
- 'second#out=>third#in2',
124
- 'third#out=>fourth#in',
125
- 'third#out=>fifth#in']
126
-
127
- puts config.to_s
128
- end
129
-
130
- it "should not allow two components with the same name" do
131
- expect do
132
- described_class.configure do |c|
133
- c.component 'first', 'First'
134
- c.component 'first', 'First'
135
- end
136
- end.to raise_error
137
- end
138
-
139
- it "should not allow two shards with the same name" do
140
- expect do
141
- described_class.configure do |c|
142
- c.shard("s1", :process => 2) { |s| }
143
- c.shard("s1", :process => 2) { |s| }
144
- end
145
- end.to raise_error
146
- end
147
-
148
- end
@@ -1,73 +0,0 @@
1
- require 'spec_helper.rb'
2
-
3
- require 'rflow/configuration'
4
-
5
-
6
- describe RFlow::Configuration do
7
- before(:each) do
8
- # RFlow::Configuration.available_data_types.clear
9
- # RFlow::Configuration.available_data_extensions.clear
10
- end
11
-
12
-
13
- describe '.add_available_data_type' do
14
- context 'if passed a data_serialization that is not avro or xml' do
15
- it "should throw an exception" do
16
- expect do
17
- RFlow::Configuration.add_available_data_type('A', 'boom', 'schema')
18
- end.to raise_error(ArgumentError)
19
- end
20
-
21
- it "should not update the available_data_types" do
22
- num_types = RFlow::Configuration.available_data_types.size
23
- RFlow::Configuration.add_available_data_type('A', 'boom', 'schema') rescue nil
24
- RFlow::Configuration.available_data_types.should have(num_types).items
25
- end
26
- end
27
- end
28
-
29
- describe "Data Extensions" do
30
-
31
- describe ".add_available_data_extension" do
32
- context 'if passed a non-module data extension' do
33
- it "should throw an exception" do
34
- expect do
35
- RFlow::Configuration.add_available_data_extension('data_type', 'not a Module')
36
- end.to raise_error(ArgumentError)
37
- end
38
- end
39
-
40
- context "if passed a valid Module as a data extension" do
41
- it "should update the available_data_extensions" do
42
- num_extensions = RFlow::Configuration.available_data_extensions['data_type'].size
43
- expect do
44
- RFlow::Configuration.add_available_data_extension('data_type', Module.new)
45
- end.to_not raise_error
46
- RFlow::Configuration.available_data_extensions['data_type'].should have(num_extensions+1).items
47
- end
48
- end
49
- end
50
-
51
- it "should perform simple 'prefix'-based inheritance for extensions" do
52
- RFlow::Configuration.add_available_data_extension('A', A = Module.new)
53
- RFlow::Configuration.add_available_data_extension('A::B', B = Module.new)
54
- RFlow::Configuration.add_available_data_extension('A::B::C', C = Module.new)
55
- RFlow::Configuration.add_available_data_extension('A::B::C::D', D = Module.new)
56
-
57
- RFlow::Configuration.available_data_extensions['A'].should have(1).item
58
- RFlow::Configuration.available_data_extensions['A'].should == [A]
59
-
60
- RFlow::Configuration.available_data_extensions['A::B'].should have(2).item
61
- RFlow::Configuration.available_data_extensions['A::B'].should == [A, B]
62
-
63
- RFlow::Configuration.available_data_extensions['A::B::C'].should have(3).item
64
- RFlow::Configuration.available_data_extensions['A::B::C'].should == [A, B, C]
65
-
66
- RFlow::Configuration.available_data_extensions['A::B::C::D'].should have(4).item
67
- RFlow::Configuration.available_data_extensions['A::B::C::D'].should == [A, B, C, D]
68
-
69
- RFlow::Configuration.available_data_extensions['A::B::C::D::E'].should have(4).item
70
- RFlow::Configuration.available_data_extensions['A::B::C::D::E'].should == [A, B, C, D]
71
- end
72
- end
73
- end