rflow 1.0.0a1 → 1.0.0a2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/Gemfile +0 -1
- data/NOTES +0 -13
- data/README.md +6 -1
- data/bin/rflow +2 -9
- data/example/basic_config.rb +1 -33
- data/example/basic_extensions.rb +0 -98
- data/example/http_config.rb +2 -3
- data/example/http_extensions.rb +6 -63
- data/lib/rflow.rb +31 -39
- data/lib/rflow/child_process.rb +112 -0
- data/lib/rflow/component.rb +77 -148
- data/lib/rflow/component/port.rb +38 -41
- data/lib/rflow/components.rb +4 -8
- data/lib/rflow/components/clock.rb +49 -0
- data/lib/rflow/components/integer.rb +39 -0
- data/lib/rflow/components/raw.rb +10 -6
- data/lib/rflow/components/replicate.rb +20 -0
- data/lib/rflow/components/ruby_proc_filter.rb +27 -0
- data/lib/rflow/configuration.rb +105 -184
- data/lib/rflow/configuration/component.rb +1 -4
- data/lib/rflow/configuration/connection.rb +11 -16
- data/lib/rflow/configuration/port.rb +3 -5
- data/lib/rflow/configuration/ruby_dsl.rb +105 -119
- data/lib/rflow/configuration/setting.rb +19 -25
- data/lib/rflow/configuration/shard.rb +1 -3
- data/lib/rflow/connection.rb +47 -10
- data/lib/rflow/connections.rb +0 -1
- data/lib/rflow/connections/zmq_connection.rb +34 -38
- data/lib/rflow/daemon_process.rb +155 -0
- data/lib/rflow/logger.rb +41 -25
- data/lib/rflow/master.rb +23 -105
- data/lib/rflow/message.rb +78 -108
- data/lib/rflow/pid_file.rb +37 -37
- data/lib/rflow/shard.rb +33 -100
- data/lib/rflow/version.rb +2 -2
- data/rflow.gemspec +2 -2
- data/schema/tick.avsc +10 -0
- data/spec/fixtures/config_ints.rb +4 -40
- data/spec/fixtures/config_shards.rb +1 -2
- data/spec/fixtures/extensions_ints.rb +0 -98
- data/spec/rflow/component/port_spec.rb +61 -0
- data/spec/rflow/components/clock_spec.rb +72 -0
- data/spec/rflow/configuration/ruby_dsl_spec.rb +150 -0
- data/spec/rflow/configuration_spec.rb +54 -0
- data/spec/rflow/forward_to_input_port_spec.rb +48 -0
- data/spec/rflow/forward_to_output_port_spec.rb +40 -0
- data/spec/rflow/logger_spec.rb +48 -0
- data/spec/rflow/message/data/raw_spec.rb +29 -0
- data/spec/rflow/message/data_spec.rb +58 -0
- data/spec/rflow/message_spec.rb +154 -0
- data/spec/rflow_spec.rb +94 -124
- data/spec/spec_helper.rb +8 -12
- metadata +46 -22
- data/lib/rflow/components/raw/extensions.rb +0 -18
- data/lib/rflow/port.rb +0 -4
- data/lib/rflow/util.rb +0 -19
- data/spec/rflow_component_port_spec.rb +0 -58
- data/spec/rflow_configuration_ruby_dsl_spec.rb +0 -148
- data/spec/rflow_configuration_spec.rb +0 -73
- data/spec/rflow_message_data_raw.rb +0 -26
- data/spec/rflow_message_data_spec.rb +0 -60
- data/spec/rflow_message_spec.rb +0 -182
- data/spec/schema_spec.rb +0 -28
- data/temp.rb +0 -295
@@ -1,26 +0,0 @@
|
|
1
|
-
require 'spec_helper.rb'
|
2
|
-
|
3
|
-
require 'rflow/components/raw'
|
4
|
-
|
5
|
-
describe 'RFlow::Message::Data::Raw Avro Schema' do
|
6
|
-
before(:each) do
|
7
|
-
@schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
|
8
|
-
end
|
9
|
-
|
10
|
-
it "should load the schema" do
|
11
|
-
@schema_string.should_not == nil
|
12
|
-
end
|
13
|
-
|
14
|
-
it "should encode and decode an object" do
|
15
|
-
raw = {'raw' => 'rawdata'}
|
16
|
-
|
17
|
-
expect {encode_avro(@schema_string, raw)}.to_not raise_error
|
18
|
-
avro_encoded_raw = encode_avro(@schema_string, raw)
|
19
|
-
|
20
|
-
expect {decode_avro(@schema_string, avro_encoded_raw)}.to_not raise_error
|
21
|
-
decoded_raw = decode_avro(@schema_string, avro_encoded_raw)
|
22
|
-
|
23
|
-
decoded_raw.should == raw
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
@@ -1,60 +0,0 @@
|
|
1
|
-
require 'spec_helper.rb'
|
2
|
-
|
3
|
-
require 'rflow/message'
|
4
|
-
|
5
|
-
describe RFlow::Message::Data do
|
6
|
-
before(:all) do
|
7
|
-
@string = 'this is a string to be serialized'
|
8
|
-
@invalid_avro_schema_string = 'invalid schema'
|
9
|
-
@valid_avro_string_schema_string = '{"type": "string"}'
|
10
|
-
@avro_serialized_string = encode_avro(@valid_avro_string_schema_string, @string)
|
11
|
-
end
|
12
|
-
|
13
|
-
context "if created without a schema" do
|
14
|
-
it "should throw an exception" do
|
15
|
-
expect {RFlow::Message::Data.new()}.to raise_error(ArgumentError)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
context "if created with an invalid schema for the serialization" do
|
20
|
-
it "should throw and exception" do
|
21
|
-
expect {RFlow::Message::Data.new(@invalid_avro_schema_string)}.to raise_error(ArgumentError)
|
22
|
-
expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
|
23
|
-
expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
context "if created with a valid avro schema and serialization" do
|
28
|
-
end
|
29
|
-
|
30
|
-
context "if created with a valid avro schema" do
|
31
|
-
it "should instantiate correctly" do
|
32
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
|
33
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
|
34
|
-
end
|
35
|
-
|
36
|
-
context "if created with a non-avro data serialization" do
|
37
|
-
it "should throw an exception" do
|
38
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'unknown')}.to raise_error(ArgumentError)
|
39
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :unknown)}.to raise_error(ArgumentError)
|
40
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'xml')}.to raise_error(ArgumentError)
|
41
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :xml)}.to raise_error(ArgumentError)
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
context "if created with an avro serialization" do
|
46
|
-
it "should instantiate correctly" do
|
47
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
|
48
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
|
49
|
-
end
|
50
|
-
|
51
|
-
context "if created with a serialized data object" do
|
52
|
-
it "should instantiate correctly" do
|
53
|
-
expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro', @avro_serialized_string)}.to_not raise_error
|
54
|
-
message = RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro', @avro_serialized_string)
|
55
|
-
p message
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
data/spec/rflow_message_spec.rb
DELETED
@@ -1,182 +0,0 @@
|
|
1
|
-
require 'spec_helper.rb'
|
2
|
-
|
3
|
-
require 'digest/md5'
|
4
|
-
|
5
|
-
require 'rflow/message'
|
6
|
-
|
7
|
-
describe RFlow::Message do
|
8
|
-
|
9
|
-
context "if created with an unknown data type" do
|
10
|
-
it "should throw an exception" do
|
11
|
-
expect {RFlow::Message.new('non_existant_data_type')}.to raise_error(ArgumentError)
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
|
16
|
-
context "if created with a known data type" do
|
17
|
-
before(:all) do
|
18
|
-
@avro_string_schema_string = '{"type": "string"}'
|
19
|
-
RFlow::Configuration.add_available_data_type(:string_type, 'avro', @avro_string_schema_string)
|
20
|
-
end
|
21
|
-
|
22
|
-
it "should instantiate correctly" do
|
23
|
-
expect {RFlow::Message.new('string_type')}.to_not raise_error
|
24
|
-
end
|
25
|
-
|
26
|
-
context "if created with empty provenance" do
|
27
|
-
context "if created with an unknown data serialization" do
|
28
|
-
it "should throw an exception" do
|
29
|
-
expect {RFlow::Message.new('string_type', [], 'unknown')}.to raise_error(ArgumentError)
|
30
|
-
expect {RFlow::Message.new('string_type', [], :unknown)}.to raise_error(ArgumentError)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
context "if created with a known data serialization" do
|
35
|
-
it "should instantiate correctly" do
|
36
|
-
expect {RFlow::Message.new('string_type', [], 'avro')}.to_not raise_error
|
37
|
-
expect {RFlow::Message.new('string_type', [], 'avro')}.to_not raise_error
|
38
|
-
end
|
39
|
-
|
40
|
-
context "if created with a mismatched schema" do
|
41
|
-
end
|
42
|
-
|
43
|
-
context "if created with a matched schema" do
|
44
|
-
end
|
45
|
-
|
46
|
-
|
47
|
-
context "if created with a nil schema" do
|
48
|
-
context "if created with a serialized data object" do
|
49
|
-
before(:all) do
|
50
|
-
@string = 'this is a string to be serialized'
|
51
|
-
@avro_serialized_string = encode_avro(@avro_string_schema_string, @string)
|
52
|
-
end
|
53
|
-
|
54
|
-
it "should instantiate correctly" do
|
55
|
-
expect {RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)}.to_not raise_error
|
56
|
-
message = RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
context "if created with invalid provenance" do
|
64
|
-
before(:all) do
|
65
|
-
@invalid_processing_event_hash = {'started_at' => 'bad time string'}
|
66
|
-
@invalid_provenance = [@invalid_processing_event_hash]
|
67
|
-
end
|
68
|
-
|
69
|
-
it "should throw an exception" do
|
70
|
-
expect {RFlow::Message.new('string_type', @invalid_provenance)}.to raise_error(ArgumentError)
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
context "if created with valid provenance" do
|
75
|
-
before(:all) do
|
76
|
-
@valid_xmlschema_time = '2001-01-01T01:01:01.000001Z'
|
77
|
-
@valid_processing_event_hash = {'component_instance_uuid' => 'uuid', 'started_at' => @valid_xmlschema_time}
|
78
|
-
@valid_processing_event = RFlow::Message::ProcessingEvent.new('uuid', @valid_xmlschema_time, @valid_xmlschema_time, 'context')
|
79
|
-
@valid_provenance = [
|
80
|
-
RFlow::Message::ProcessingEvent.new('uuid'),
|
81
|
-
@valid_processing_event_hash,
|
82
|
-
@valid_processing_event,
|
83
|
-
]
|
84
|
-
@valid_provenance_hashes = [
|
85
|
-
{"component_instance_uuid"=>"uuid", "started_at"=>nil, "completed_at"=>nil, "context"=>nil},
|
86
|
-
{"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>nil, "context"=>nil},
|
87
|
-
{"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>@valid_xmlschema_time, "context"=>"context"},
|
88
|
-
]
|
89
|
-
end
|
90
|
-
|
91
|
-
it "should instantiate correctly" do
|
92
|
-
p @valid_provenance
|
93
|
-
expect {RFlow::Message.new('string_type', @valid_provenance)}.to_not raise_error
|
94
|
-
end
|
95
|
-
|
96
|
-
it "should correctly set the provenance processing events" do
|
97
|
-
message = RFlow::Message.new('string_type', @valid_provenance)
|
98
|
-
message.provenance[1].component_instance_uuid.should == 'uuid'
|
99
|
-
message.provenance[1].started_at.should == Time.xmlschema(@valid_xmlschema_time)
|
100
|
-
message.provenance[1].completed_at.should == nil
|
101
|
-
message.provenance[1].context.should == nil
|
102
|
-
end
|
103
|
-
|
104
|
-
it "should to_hash its provenance correctly" do
|
105
|
-
message = RFlow::Message.new('string_type', @valid_provenance)
|
106
|
-
message.provenance.map(&:to_hash).should == @valid_provenance_hashes
|
107
|
-
end
|
108
|
-
|
109
|
-
end
|
110
|
-
|
111
|
-
context "if correctly created" do
|
112
|
-
it "should serialize and deserialized correctly to/from avro" do
|
113
|
-
message = RFlow::Message.new('string_type')
|
114
|
-
message.provenance << RFlow::Message::ProcessingEvent.new('UUID')
|
115
|
-
message.data.data_object = 'teh awesome'
|
116
|
-
|
117
|
-
processed_message = RFlow::Message.from_avro(message.to_avro)
|
118
|
-
message.data.to_avro.should == processed_message.data.to_avro
|
119
|
-
message.data.data_object.should == processed_message.data.data_object
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|
123
|
-
context "if data extensions exist" do
|
124
|
-
it "should extend the data element with the extension" do
|
125
|
-
module ExtensionModule; def ext_method; end; end
|
126
|
-
|
127
|
-
message = RFlow::Message.new('string_type')
|
128
|
-
message.data.methods.should_not include(:ext_method)
|
129
|
-
|
130
|
-
RFlow::Configuration.add_available_data_extension('string_type', ExtensionModule)
|
131
|
-
message = RFlow::Message.new('string_type')
|
132
|
-
message.data.methods.should include(:ext_method)
|
133
|
-
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
|
138
|
-
it "should correctly handle large raw types" do
|
139
|
-
message = RFlow::Message.new('RFlow::Message::Data::Raw')
|
140
|
-
message.data.raw = Array.new(101) { rand(256) }.pack('c*')
|
141
|
-
|
142
|
-
message_avro = message.to_avro.force_encoding('BINARY')
|
143
|
-
|
144
|
-
processed_message = RFlow::Message.from_avro(message_avro)
|
145
|
-
processed_message_avro = processed_message.to_avro.force_encoding('BINARY')
|
146
|
-
|
147
|
-
@raw_schema = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
|
148
|
-
|
149
|
-
encode_avro(@raw_schema, message.data.data_object).should == message.data.to_avro
|
150
|
-
decode_avro(@raw_schema, message.data.to_avro).should == message.data.data_object
|
151
|
-
|
152
|
-
p message.data.raw
|
153
|
-
p message_avro
|
154
|
-
p message_avro.bytesize
|
155
|
-
p processed_message_avro
|
156
|
-
p processed_message_avro.bytesize
|
157
|
-
|
158
|
-
p message_avro.encoding
|
159
|
-
p message_avro.valid_encoding?
|
160
|
-
|
161
|
-
p processed_message_avro.encoding
|
162
|
-
p processed_message_avro.valid_encoding?
|
163
|
-
|
164
|
-
message_data_avro = message.data.to_avro.force_encoding('BINARY')
|
165
|
-
processed_message_data_avro = processed_message.data.to_avro.force_encoding('BINARY')
|
166
|
-
|
167
|
-
p message_data_avro.encoding
|
168
|
-
p message_data_avro.valid_encoding?
|
169
|
-
p message_data_avro
|
170
|
-
p processed_message_data_avro.encoding
|
171
|
-
p processed_message_data_avro.valid_encoding?
|
172
|
-
p processed_message_data_avro
|
173
|
-
|
174
|
-
Digest::MD5.hexdigest(message_avro).should == Digest::MD5.hexdigest(processed_message_avro)
|
175
|
-
|
176
|
-
|
177
|
-
message_data_avro.should == processed_message_data_avro
|
178
|
-
Digest::MD5.hexdigest(message_data_avro).should == Digest::MD5.hexdigest(processed_message_data_avro)
|
179
|
-
Digest::MD5.hexdigest(message.data.raw).should == Digest::MD5.hexdigest(processed_message.data.raw)
|
180
|
-
end
|
181
|
-
|
182
|
-
end
|
data/spec/schema_spec.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
require 'spec_helper.rb'
|
2
|
-
|
3
|
-
describe 'RFlow::Message::Data::Raw Avro Schema' do
|
4
|
-
before(:each) do
|
5
|
-
@schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
|
6
|
-
end
|
7
|
-
|
8
|
-
it "should encode and decode an object" do
|
9
|
-
raw = {
|
10
|
-
'raw' => Array.new(256) { rand(256) }.pack('c*')
|
11
|
-
}
|
12
|
-
|
13
|
-
expect {encode_avro(@schema_string, raw)}.to_not raise_error
|
14
|
-
avro_encoded_raw = encode_avro(@schema_string, raw)
|
15
|
-
|
16
|
-
expect {decode_avro(@schema_string, avro_encoded_raw)}.to_not raise_error
|
17
|
-
decoded_raw = decode_avro(@schema_string, avro_encoded_raw)
|
18
|
-
|
19
|
-
decoded_raw.should == raw
|
20
|
-
|
21
|
-
p decoded_raw['raw'].encoding
|
22
|
-
p raw['raw'].encoding
|
23
|
-
|
24
|
-
decoded_raw['raw'].should == raw['raw']
|
25
|
-
|
26
|
-
end
|
27
|
-
|
28
|
-
end
|
data/temp.rb
DELETED
@@ -1,295 +0,0 @@
|
|
1
|
-
module RFlow
|
2
|
-
def self.run(config)
|
3
|
-
# Take in the config file
|
4
|
-
# Set a module-level config
|
5
|
-
# Set module-level attributes (logger)
|
6
|
-
# Create manager
|
7
|
-
# Start manager with parsed config elements
|
8
|
-
end
|
9
|
-
|
10
|
-
class Manager
|
11
|
-
|
12
|
-
def initialize(config)
|
13
|
-
end
|
14
|
-
# Find each component
|
15
|
-
# Instantiate (process management)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
class SchemaRegistry
|
20
|
-
# maps data type names to schemas based on schema type
|
21
|
-
find_by_data_type_name
|
22
|
-
end
|
23
|
-
|
24
|
-
class MessageDataRegistry
|
25
|
-
def find(data_type_name)
|
26
|
-
# returns a data type class if registered, nil otherwise
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
class Message::Data
|
31
|
-
# contains the schema + data information
|
32
|
-
# subclasses can add extra functionality, otherwise will just have
|
33
|
-
# acces to standard messagedata stuffs (i.e. standard avro data types)
|
34
|
-
# delegates a lot to standard Avro types
|
35
|
-
|
36
|
-
# how does this get access to the registry at the class level?
|
37
|
-
class << self
|
38
|
-
attr_accessor :class_registry
|
39
|
-
attr_accessor :schema_registry
|
40
|
-
end
|
41
|
-
|
42
|
-
# Pointer to encapsulating message
|
43
|
-
attr_accessor :message
|
44
|
-
|
45
|
-
def initialize(data_type_name, serialized_data=nil, schema_name=nil, schema_type=nil, schema=nil, message=nil)
|
46
|
-
# schema_name ||= 'org.rflow.Messages.GenericStringMap'
|
47
|
-
# schema_type ||= 'avro'
|
48
|
-
# schema ||= 'default avro schema'
|
49
|
-
|
50
|
-
merge_options
|
51
|
-
|
52
|
-
# TODO: think about schema resolution and conflicts between passed
|
53
|
-
# data and schema registry
|
54
|
-
# Lookup schema based on data type name
|
55
|
-
registered_schema_name, registered_schema, registered_schema_type = self.class.schema_registry.find(data_type_name)
|
56
|
-
if registered_schema.nil? && schema
|
57
|
-
# If you were given a schema and didn't get one from the
|
58
|
-
# registry register the schema?
|
59
|
-
self.class.schema_registry.register(data_type_name, schema_name, schema_type, schema)
|
60
|
-
else
|
61
|
-
|
62
|
-
end
|
63
|
-
|
64
|
-
end
|
65
|
-
|
66
|
-
def self.create(data_type_name, data=nil, schema_name=nil, schema_type=nil, schema=nil)
|
67
|
-
# look for object in registry by data_type_name
|
68
|
-
# if object found, call new on that object
|
69
|
-
# otherwise, call new on the default object
|
70
|
-
message_class = self.class.data_class_registry.find(data_type_name)
|
71
|
-
if message_class.nil?
|
72
|
-
MessageData.new(data_type_name, data, schema_name, schema_type, schema)
|
73
|
-
else
|
74
|
-
message_class.create(data_type_name, data, schema_name, schema_type, schema)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
module HTTPResponse
|
80
|
-
end
|
81
|
-
|
82
|
-
Message.new.extend(HTTPResponse)
|
83
|
-
|
84
|
-
class HTTPRequest < RFlow::Message::Data
|
85
|
-
# used to add methods, defaults, and more to data object, if required
|
86
|
-
|
87
|
-
# Put this in the registry
|
88
|
-
AVRO_SCHEMA_NAME = 'org.rflow.http_request'
|
89
|
-
DATA_TYPE_NAME = "HTTPRequest"
|
90
|
-
|
91
|
-
# All subclasses must have the same initialize signature. They need
|
92
|
-
# to figure out what to do when they get the extra parameters that
|
93
|
-
# might conflict with expectations. Subclasses are usually meant to
|
94
|
-
# enable extra functionality on a given data type, so as long as it
|
95
|
-
# operates properly, it might not care (duck typing)
|
96
|
-
def initialize(data_type_name, data, schema_name, schema_type, schema)
|
97
|
-
super(DATA_TYPE_NAME, data, AVRO_SCHEMA_NAME)
|
98
|
-
# do nice stuff with data here
|
99
|
-
end
|
100
|
-
|
101
|
-
def self.create(data_type_name, data, schema_name, schema_type, schema)
|
102
|
-
# figure out if you are being called with incompatible arguments,
|
103
|
-
# i.e. schema stuff
|
104
|
-
end
|
105
|
-
|
106
|
-
end
|
107
|
-
|
108
|
-
class Message
|
109
|
-
# contains all definitions about what to do for a message
|
110
|
-
# has a default Avro schema for a data type
|
111
|
-
|
112
|
-
class << self
|
113
|
-
attr_accessor :data_class_registry
|
114
|
-
end
|
115
|
-
|
116
|
-
|
117
|
-
# Should load all the data stuff, perhaps to top level method on object
|
118
|
-
|
119
|
-
attr_accessor :data_type_name, :provenance, :origination_context, :data_type_schema, :data
|
120
|
-
|
121
|
-
def initialize(data_type_name, provenance=nil, origination_context=nil, data_type_schema=nil, data=nil)
|
122
|
-
if data
|
123
|
-
# Potentially register this data_type_name to the schema
|
124
|
-
else
|
125
|
-
# Lookup MessageData type in the MessageDataRegistry
|
126
|
-
# if found and a class, create a specific MessageData object
|
127
|
-
# extend it with the module
|
128
|
-
# else, create generic MessageData object which will use
|
129
|
-
# the schema registry, under the hood
|
130
|
-
# if found and a module, extend object with found module
|
131
|
-
|
132
|
-
message_data_class = self.class.data_class_registry.find(data_type_name)
|
133
|
-
if message_data_class && message_data_class.class.is_a? Class
|
134
|
-
message_data = message_data_class.new
|
135
|
-
else
|
136
|
-
message_data = Message::Data.new
|
137
|
-
message_data.extend message_data_class if message_data_class.is_a? Module
|
138
|
-
end
|
139
|
-
end
|
140
|
-
end
|
141
|
-
|
142
|
-
end
|
143
|
-
|
144
|
-
class Port
|
145
|
-
def read_message
|
146
|
-
parts = read_all_parts
|
147
|
-
parts.assemble
|
148
|
-
data_type_name = read_message_part
|
149
|
-
provenance = read_message_part
|
150
|
-
origination_context = read_message_part
|
151
|
-
data_type_schema = read_message_part
|
152
|
-
data = read_message_part
|
153
|
-
|
154
|
-
message = Message.new(data_type_name, provenance, origination_context, data_type_schema, data)
|
155
|
-
|
156
|
-
message
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
class PortCollection
|
161
|
-
end
|
162
|
-
|
163
|
-
class Logger
|
164
|
-
end
|
165
|
-
|
166
|
-
class Component
|
167
|
-
def self.input_port(port_def)
|
168
|
-
@@input_ports ||= PortCollection.new
|
169
|
-
if port_def.is_a? Array
|
170
|
-
port_name = port_def.first.to_sym
|
171
|
-
port_incidence = :array
|
172
|
-
else
|
173
|
-
port_name = port_def
|
174
|
-
port_incidence = :single
|
175
|
-
end
|
176
|
-
@@input_ports[port_name] = InputPort.new port_name, port_incidence
|
177
|
-
end
|
178
|
-
|
179
|
-
def self.output_port
|
180
|
-
# same as input port with different stuffs
|
181
|
-
end
|
182
|
-
|
183
|
-
STATES = [:initialized, :started, :configured, :running, :stopping, :stopped]
|
184
|
-
attr_accessor :state
|
185
|
-
attr_accessor :input_ports
|
186
|
-
attr_accessor :output_ports
|
187
|
-
|
188
|
-
attr_accessor :uuid
|
189
|
-
attr_accessor :name
|
190
|
-
|
191
|
-
CONFIG_DEFAULTS = {
|
192
|
-
:logger,
|
193
|
-
:working_directory_path,
|
194
|
-
}
|
195
|
-
|
196
|
-
def initialize(config, run_directory)
|
197
|
-
# configure component
|
198
|
-
config = {
|
199
|
-
}
|
200
|
-
|
201
|
-
# TODO: where is the management bus listener configured/started
|
202
|
-
end
|
203
|
-
|
204
|
-
def run
|
205
|
-
input_ports.ready do |port|
|
206
|
-
message = port.read_message
|
207
|
-
process_input(port, message)
|
208
|
-
# read from the port and think about things
|
209
|
-
out.send('stuff')
|
210
|
-
another_out.send('more stuff')
|
211
|
-
end
|
212
|
-
# listen to
|
213
|
-
end
|
214
|
-
|
215
|
-
def process_message(input_port, message)
|
216
|
-
|
217
|
-
end
|
218
|
-
|
219
|
-
def receive_message(port)
|
220
|
-
port.receive
|
221
|
-
end
|
222
|
-
|
223
|
-
def send_message(port, message)
|
224
|
-
port.send(message)
|
225
|
-
end
|
226
|
-
|
227
|
-
end
|
228
|
-
|
229
|
-
class HTTPServer < RFlow::Component
|
230
|
-
input_port :responses
|
231
|
-
output_port :requests
|
232
|
-
|
233
|
-
input_types "HTTP::Response"
|
234
|
-
output_types "HTTP::Request"
|
235
|
-
|
236
|
-
|
237
|
-
end
|
238
|
-
|
239
|
-
class PassThrough < RFlow::Component
|
240
|
-
input_port [:in]
|
241
|
-
input_port :another_in
|
242
|
-
output_port :out
|
243
|
-
output_port :another_out
|
244
|
-
|
245
|
-
output_types
|
246
|
-
|
247
|
-
def initialize(config, run_directory)
|
248
|
-
# This will initialize the ports
|
249
|
-
super
|
250
|
-
# Do stuff to initialize component. Don't assume singleton
|
251
|
-
end
|
252
|
-
|
253
|
-
|
254
|
-
def process_message(input_port, data)
|
255
|
-
out.send(message)
|
256
|
-
another_out.send(message)
|
257
|
-
|
258
|
-
|
259
|
-
end
|
260
|
-
|
261
|
-
def process_data(input_port
|
262
|
-
|
263
|
-
end
|
264
|
-
|
265
|
-
|
266
|
-
class Transform < RFlow::Component
|
267
|
-
|
268
|
-
end
|
269
|
-
|
270
|
-
# Plugins:
|
271
|
-
|
272
|
-
# MessageData subclass: rflow-data-http_request
|
273
|
-
# lib/rflow-data-http_request.rb
|
274
|
-
require 'rflow'
|
275
|
-
require 'lib/data_name'
|
276
|
-
RFlow.available_data_types << data_name_object
|
277
|
-
|
278
|
-
|
279
|
-
# Component: rflow-component-http_server
|
280
|
-
# lib/rflow-component-http_server
|
281
|
-
require 'rflow'
|
282
|
-
require 'lib/component_name'
|
283
|
-
RFlow.available_components << component_class
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
# lib/component_name.rb ->
|
289
|
-
# data_type_name => schema + registration: just register in the application
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
# Server -> (HttpRequest -> Translate -> HTTPResponse) -> Server
|
294
|
-
|
295
|
-
|