rflow 1.0.0a2 → 1.0.0a3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/README.VAGRANT +63 -0
- data/README.md +118 -33
- data/Vagrantfile +53 -0
- data/bin/rflow +6 -1
- data/example/basic_extensions.rb +7 -8
- data/example/http_extensions.rb +7 -8
- data/lib/rflow/broker.rb +18 -0
- data/lib/rflow/child_process.rb +3 -1
- data/lib/rflow/component.rb +51 -61
- data/lib/rflow/component/port.rb +24 -15
- data/lib/rflow/configuration.rb +1 -0
- data/lib/rflow/configuration/connection.rb +35 -17
- data/lib/rflow/configuration/ruby_dsl.rb +47 -9
- data/lib/rflow/connection.rb +13 -9
- data/lib/rflow/connections/zmq_connection.rb +46 -3
- data/lib/rflow/daemon_process.rb +1 -1
- data/lib/rflow/master.rb +8 -1
- data/lib/rflow/shard.rb +8 -2
- data/lib/rflow/version.rb +1 -1
- data/rflow.gemspec +6 -6
- data/spec/fixtures/extensions_ints.rb +7 -8
- data/spec/rflow/component/port_spec.rb +16 -22
- data/spec/rflow/components/clock_spec.rb +12 -17
- data/spec/rflow/configuration/ruby_dsl_spec.rb +234 -46
- data/spec/rflow/configuration_spec.rb +5 -5
- data/spec/rflow/forward_to_input_port_spec.rb +10 -18
- data/spec/rflow/forward_to_output_port_spec.rb +6 -13
- data/spec/rflow/logger_spec.rb +6 -6
- data/spec/rflow/message/data/raw_spec.rb +3 -3
- data/spec/rflow/message_spec.rb +16 -16
- data/spec/rflow_spec.rb +37 -37
- data/spec/spec_helper.rb +3 -5
- metadata +20 -17
data/lib/rflow/broker.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'rflow/child_process'
|
2
|
+
|
3
|
+
class RFlow
|
4
|
+
# A message broker to mediate messages along a connection.
|
5
|
+
# The broker runs in a child process and will not return from spawn!.
|
6
|
+
class Broker < ChildProcess
|
7
|
+
class << self
|
8
|
+
def build(config)
|
9
|
+
case config.class.name
|
10
|
+
when 'RFlow::Configuration::ZMQStreamer'
|
11
|
+
RFlow::Connections::ZMQStreamer.new(config)
|
12
|
+
else
|
13
|
+
raise ArgumentError, 'Only ZMQ brokers currently supported'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/lib/rflow/child_process.rb
CHANGED
data/lib/rflow/component.rb
CHANGED
@@ -24,16 +24,7 @@ class RFlow
|
|
24
24
|
|
25
25
|
# Create the port accessor method based on the port name
|
26
26
|
define_method name.to_s.to_sym do
|
27
|
-
|
28
|
-
return port if port
|
29
|
-
|
30
|
-
# If the port was not connected, return a port-like object
|
31
|
-
# that can respond/log but doesn't send any data. Note,
|
32
|
-
# it won't be available in the 'by_uuid' collection, as it
|
33
|
-
# doesn't have a configured uuid
|
34
|
-
RFlow.logger.debug "'#{self.name}##{name}' not connected, creating a disconnected port"
|
35
|
-
|
36
|
-
DisconnectedPort.new(OpenStruct.new(:name => name, :uuid => 0)).tap {|d| ports << d }
|
27
|
+
ports.by_name[name.to_s]
|
37
28
|
end
|
38
29
|
end
|
39
30
|
|
@@ -50,33 +41,49 @@ class RFlow
|
|
50
41
|
|
51
42
|
RFlow.logger.debug "Instantiating component '#{config.name}' as '#{config.specification}' (#{config.uuid})"
|
52
43
|
begin
|
53
|
-
|
44
|
+
component_class = RFlow.configuration.available_components[config.specification]
|
54
45
|
|
55
|
-
if
|
46
|
+
if component_class
|
56
47
|
RFlow.logger.debug "Component found in configuration.available_components['#{config.specification}']"
|
57
|
-
component.new(config)
|
58
48
|
else
|
59
49
|
RFlow.logger.debug "Component not found in configuration.available_components, constantizing component '#{config.specification}'"
|
60
|
-
config.specification.constantize
|
50
|
+
component_class = config.specification.constantize
|
51
|
+
end
|
52
|
+
|
53
|
+
component_class.new(uuid: config.uuid, name: config.name).tap do |component|
|
54
|
+
config.input_ports.each {|p| component.configure_input_port! p.name, uuid: p.uuid }
|
55
|
+
config.output_ports.each {|p| component.configure_output_port! p.name, uuid: p.uuid }
|
56
|
+
|
57
|
+
config.input_ports.each do |p|
|
58
|
+
p.input_connections.each do |c|
|
59
|
+
component.send(p.name.to_sym).add_connection c.input_port_key, Connection.build(c)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
config.output_ports.each do |p|
|
64
|
+
p.output_connections.each do |c|
|
65
|
+
component.send(p.name.to_sym).add_connection c.output_port_key, Connection.build(c)
|
66
|
+
end
|
67
|
+
end
|
61
68
|
end
|
62
69
|
rescue NameError => e
|
63
|
-
raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}'
|
70
|
+
raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): the class '#{config.specification}' could not be loaded (#{e.message})"
|
64
71
|
rescue Exception => e
|
65
|
-
raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}"
|
72
|
+
raise RuntimeError, "Could not instantiate component '#{config.name}' as '#{config.specification}' (#{config.uuid}): #{e.class} #{e.message}, because: #{e.backtrace.inspect}"
|
66
73
|
end
|
67
74
|
end
|
68
75
|
end
|
69
76
|
|
70
|
-
|
77
|
+
attr_accessor :uuid, :name
|
78
|
+
attr_reader :ports
|
71
79
|
|
72
|
-
def initialize(
|
73
|
-
@
|
74
|
-
@uuid =
|
75
|
-
@name = config.name
|
80
|
+
def initialize(args = {})
|
81
|
+
@name = args[:name]
|
82
|
+
@uuid = args[:uuid]
|
76
83
|
@ports = PortCollection.new
|
77
84
|
|
78
|
-
|
79
|
-
|
85
|
+
self.class.defined_input_ports.each {|name, _| ports << InputPort.new(self, name: name) }
|
86
|
+
self.class.defined_output_ports.each {|name, _| ports << OutputPort.new(self, name: name) }
|
80
87
|
end
|
81
88
|
|
82
89
|
# Returns a list of connected input ports. Each port will have
|
@@ -87,15 +94,33 @@ class RFlow
|
|
87
94
|
# one or more keys associated with the particular connection.
|
88
95
|
def output_ports; ports.by_type["RFlow::Component::OutputPort"]; end
|
89
96
|
|
90
|
-
|
91
|
-
|
97
|
+
def configure_input_port!(port_name, options = {})
|
98
|
+
RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) input port '#{port_name}' (#{options[:uuid]})"
|
99
|
+
unless self.class.defined_input_ports.include? port_name
|
100
|
+
raise ArgumentError, "Input port '#{port_name}' not defined on component '#{self.class}'"
|
101
|
+
end
|
102
|
+
ports.by_name[port_name].uuid = options[:uuid]
|
103
|
+
end
|
104
|
+
|
105
|
+
def configure_output_port!(port_name, options = {})
|
106
|
+
RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) output port '#{port_name}' (#{options[:uuid]})"
|
107
|
+
unless self.class.defined_output_ports.include? port_name
|
108
|
+
raise ArgumentError, "Output port '#{port_name}' not defined on component '#{self.class}'"
|
109
|
+
end
|
110
|
+
ports.by_name[port_name].uuid = options[:uuid]
|
111
|
+
end
|
92
112
|
|
93
113
|
# Tell the component to establish its ports' connections, i.e. make
|
94
114
|
# the connection. Uses the underlying connection object. Also
|
95
115
|
# establishes the callbacks for each of the input ports
|
96
|
-
def
|
116
|
+
def connect_inputs!
|
97
117
|
input_ports.each {|port| port.recv_callback = method(:process_message) }
|
98
118
|
input_ports.each(&:connect!)
|
119
|
+
end
|
120
|
+
|
121
|
+
# Tell the component to establish its ports' connections, i.e. make
|
122
|
+
# the connection. Uses the underlying connection object.
|
123
|
+
def connect_outputs!
|
99
124
|
output_ports.each(&:connect!)
|
100
125
|
end
|
101
126
|
|
@@ -137,40 +162,5 @@ class RFlow
|
|
137
162
|
# before the global RFlow exit. Sublcasses should implement to
|
138
163
|
# cleanup any leftover state, e.g. flush file handles, etc
|
139
164
|
def cleanup!; end
|
140
|
-
|
141
|
-
private
|
142
|
-
def configure_ports!
|
143
|
-
@config.input_ports.each do |p|
|
144
|
-
RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with input port '#{p.name}' (#{p.uuid})"
|
145
|
-
unless self.class.defined_input_ports.include? p.name
|
146
|
-
raise ArgumentError, "Input port '#{p.name}' not defined on component '#{self.class}'"
|
147
|
-
end
|
148
|
-
ports << InputPort.new(p)
|
149
|
-
end
|
150
|
-
|
151
|
-
@config.output_ports.each do |p|
|
152
|
-
RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) with output port '#{p.name}' (#{p.uuid})"
|
153
|
-
unless self.class.defined_output_ports.include? p.name
|
154
|
-
raise ArgumentError, "Output port '#{p.name}' not defined on component '#{self.class}'"
|
155
|
-
end
|
156
|
-
ports << OutputPort.new(p)
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
|
-
def configure_connections!
|
161
|
-
@config.input_ports.each do |p|
|
162
|
-
p.input_connections.each do |c|
|
163
|
-
RFlow.logger.debug "Configuring input port '#{p.name}' (#{p.uuid}) key '#{c.input_port_key}' with #{c.type.to_s} connection '#{c.name}' (#{c.uuid})"
|
164
|
-
ports.by_uuid[p.uuid].add_connection c.input_port_key, Connection.build(c)
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
@config.output_ports.each do |p|
|
169
|
-
p.output_connections.each do |c|
|
170
|
-
RFlow.logger.debug "Configuring output port '#{p.name}' (#{p.uuid}) key '#{c.output_port_key}' with #{c.type.to_s} connection '#{c.name}' (#{c.uuid})"
|
171
|
-
ports.by_uuid[p.uuid].add_connection c.output_port_key, Connection.build(c)
|
172
|
-
end
|
173
|
-
end
|
174
|
-
end
|
175
165
|
end
|
176
166
|
end
|
data/lib/rflow/component/port.rb
CHANGED
@@ -9,28 +9,25 @@ class RFlow
|
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
|
-
# Collection class to make it easier to index by both names
|
13
|
-
#
|
12
|
+
# Collection class to make it easier to index by both names
|
13
|
+
# and types.
|
14
14
|
class PortCollection
|
15
|
-
attr_reader :ports, :
|
15
|
+
attr_reader :ports, :by_name, :by_type
|
16
16
|
|
17
17
|
def initialize
|
18
18
|
@ports = []
|
19
|
-
@by_uuid = {}
|
20
19
|
@by_name = {}
|
21
20
|
@by_type = Hash.new {|hash, key| hash[key.to_s] = []}
|
22
21
|
end
|
23
22
|
|
24
23
|
def <<(port)
|
25
|
-
by_uuid[port.uuid.to_s] = port
|
26
24
|
by_name[port.name.to_s] = port
|
27
25
|
by_type[port.class.to_s] << port
|
28
26
|
ports << port
|
29
27
|
self
|
30
28
|
end
|
31
29
|
|
32
|
-
# Enumerate through each
|
33
|
-
# referenced) port
|
30
|
+
# Enumerate through each port
|
34
31
|
# TODO: simplify with enumerators and procs
|
35
32
|
def each
|
36
33
|
ports.each {|port| yield port }
|
@@ -38,7 +35,12 @@ class RFlow
|
|
38
35
|
end
|
39
36
|
|
40
37
|
class Port
|
41
|
-
attr_reader :connected
|
38
|
+
attr_reader :connected, :component
|
39
|
+
|
40
|
+
def initialize(component)
|
41
|
+
@component = component
|
42
|
+
end
|
43
|
+
|
42
44
|
def connected?; connected; end
|
43
45
|
end
|
44
46
|
|
@@ -49,16 +51,16 @@ class RFlow
|
|
49
51
|
# result in the same message being sent to all indexed
|
50
52
|
# connections.
|
51
53
|
class HashPort < Port
|
52
|
-
|
54
|
+
attr_accessor :name, :uuid
|
53
55
|
|
54
56
|
protected
|
55
57
|
attr_reader :connections_for
|
56
58
|
|
57
59
|
public
|
58
|
-
def initialize(
|
59
|
-
|
60
|
-
|
61
|
-
|
60
|
+
def initialize(component, args = {})
|
61
|
+
super(component)
|
62
|
+
self.uuid = args[:uuid]
|
63
|
+
self.name = args[:name]
|
62
64
|
@connections_for = Hash.new {|hash, key| hash[key] = [].extend(ConnectionCollection)}
|
63
65
|
end
|
64
66
|
|
@@ -77,9 +79,18 @@ class RFlow
|
|
77
79
|
|
78
80
|
# Adds a connection for a given key
|
79
81
|
def add_connection(key, connection)
|
82
|
+
RFlow.logger.debug "Attaching #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) to port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
|
80
83
|
connections_for[key] << connection
|
81
84
|
end
|
82
85
|
|
86
|
+
def direct_connect(other_port)
|
87
|
+
case other_port
|
88
|
+
when InputPort; add_connection nil, ForwardToInputPort.new(other_port)
|
89
|
+
when OutputPort; add_connection nil, ForwardToOutputPort.new(other_port)
|
90
|
+
else raise ArgumentError, "Unknown port type #{other_port.class.name}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
83
94
|
# Return a list of connected keys
|
84
95
|
def keys
|
85
96
|
connections_for.keys
|
@@ -142,7 +153,5 @@ class RFlow
|
|
142
153
|
all_connections.send_message(message)
|
143
154
|
end
|
144
155
|
end
|
145
|
-
|
146
|
-
class DisconnectedPort < HashPort; end
|
147
156
|
end
|
148
157
|
end
|
data/lib/rflow/configuration.rb
CHANGED
@@ -211,6 +211,7 @@ class RFlow
|
|
211
211
|
def [](name); Setting.find_by_name(name).value rescue nil; end
|
212
212
|
def settings; Setting.all; end
|
213
213
|
def shards; Shard.all; end
|
214
|
+
def connections; Connection.all; end
|
214
215
|
def shard(uuid); Shard.find_by_uuid uuid; end
|
215
216
|
def components; Component.all; end
|
216
217
|
def component(uuid); Component.find_by_uuid uuid; end
|
@@ -46,6 +46,9 @@ class RFlow
|
|
46
46
|
# allow defaults to use other parameters in the connection to
|
47
47
|
# construct the appropriate default value.
|
48
48
|
def self.default_options; {}; end
|
49
|
+
|
50
|
+
# By default, no broker processes are required to manage a connection.
|
51
|
+
def brokers; []; end
|
49
52
|
end
|
50
53
|
|
51
54
|
# STI Subclass for ZMQ connections and their required options
|
@@ -62,31 +65,46 @@ class RFlow
|
|
62
65
|
end
|
63
66
|
end
|
64
67
|
|
65
|
-
# STI Subclass for
|
66
|
-
|
68
|
+
# STI Subclass for brokered ZMQ connections and their required options
|
69
|
+
#
|
70
|
+
# We name the IPCs to resemble a quasi-component. Outputting to this
|
71
|
+
# connection goes to the 'in' of the IPC pair. Reading input from this
|
72
|
+
# connection comes from the 'out' of the IPC pair.
|
73
|
+
#
|
74
|
+
# The broker shuttles messages between the two to support the many-to-many
|
75
|
+
# delivery pattern.
|
76
|
+
class BrokeredZMQConnection < Connection
|
67
77
|
def self.default_options
|
68
78
|
{
|
69
|
-
'
|
70
|
-
'
|
71
|
-
'
|
72
|
-
'
|
73
|
-
'
|
74
|
-
'
|
75
|
-
|
76
|
-
# If a queue is created, these are the default parameters
|
77
|
-
# for said queue type
|
78
|
-
'queue_passive' => false,
|
79
|
-
'queue_durable' => true,
|
80
|
-
'queue_exclusive' => false,
|
81
|
-
'queue_auto_delete' => false,
|
82
|
-
'queue_nowait' => true,
|
79
|
+
'output_socket_type' => 'PUSH',
|
80
|
+
'output_address' => lambda{|conn| "ipc://rflow.#{conn.uuid}.in"},
|
81
|
+
'output_responsibility' => 'connect',
|
82
|
+
'input_socket_type' => 'PULL',
|
83
|
+
'input_address' => lambda{|conn| "ipc://rflow.#{conn.uuid}.out"},
|
84
|
+
'input_responsibility' => 'connect',
|
83
85
|
}
|
84
86
|
end
|
87
|
+
|
88
|
+
# A brokered ZMQ connection requires one broker process.
|
89
|
+
def brokers
|
90
|
+
@brokers ||= [ZMQStreamer.new(self)]
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# Represents the broker process configuration. No special parameters
|
95
|
+
# that can't be derived from the connection. Not persisted in the database -
|
96
|
+
# it's encapsulated in the nature of the connection.
|
97
|
+
class ZMQStreamer
|
98
|
+
attr_reader :connection
|
99
|
+
|
100
|
+
def initialize(connection)
|
101
|
+
@connection = connection
|
102
|
+
end
|
85
103
|
end
|
86
104
|
|
87
105
|
# for testing purposes
|
88
106
|
class NullConfiguration
|
89
|
-
attr_accessor :name, :uuid, :options
|
107
|
+
attr_accessor :name, :uuid, :options, :input_port_key, :output_port_key
|
90
108
|
end
|
91
109
|
end
|
92
110
|
end
|
@@ -42,6 +42,16 @@ class RFlow
|
|
42
42
|
@current_shard = default_shard
|
43
43
|
end
|
44
44
|
|
45
|
+
# shortcut
|
46
|
+
def process(name, options = {}, &block)
|
47
|
+
shard(name, options.merge(:type => :process), &block)
|
48
|
+
end
|
49
|
+
|
50
|
+
# shortcut
|
51
|
+
def thread(name, options = {}, &block)
|
52
|
+
shard(name, options.merge(:type => :thread), &block)
|
53
|
+
end
|
54
|
+
|
45
55
|
# DSL method to specify a component. Expects a name,
|
46
56
|
# specification, and set of component specific options, that
|
47
57
|
# must be marshallable into the database (i.e. should all be strings)
|
@@ -85,12 +95,12 @@ class RFlow
|
|
85
95
|
def self.configure
|
86
96
|
config_file = self.new
|
87
97
|
yield config_file
|
88
|
-
config_file.
|
98
|
+
config_file.process_objects
|
89
99
|
end
|
90
100
|
|
91
101
|
# Method to process the 'DSL' objects into the config database
|
92
102
|
# via ActiveRecord
|
93
|
-
def
|
103
|
+
def process_objects
|
94
104
|
process_setting_specs
|
95
105
|
process_shard_specs
|
96
106
|
process_connection_specs
|
@@ -154,8 +164,8 @@ class RFlow
|
|
154
164
|
|
155
165
|
# For each given connection, break up each input/output
|
156
166
|
# component/port specification, ensure that the component
|
157
|
-
# already exists in the database (by name).
|
158
|
-
#
|
167
|
+
# already exists in the database (by name). Chooses the best
|
168
|
+
# connection type for any pair of components.
|
159
169
|
def process_connection_specs
|
160
170
|
connection_specs.each do |spec|
|
161
171
|
begin
|
@@ -175,11 +185,39 @@ class RFlow
|
|
175
185
|
input_port = input_component.input_ports.find_or_initialize_by_name :name => spec[:input_port_name]
|
176
186
|
input_port.save!
|
177
187
|
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
188
|
+
output_shards = output_component.shard.count
|
189
|
+
input_shards = input_component.shard.count
|
190
|
+
|
191
|
+
in_shard_connection = output_component.shard == input_component.shard
|
192
|
+
one_to_one = output_shards == 1 && input_shards == 1
|
193
|
+
one_to_many = output_shards == 1 && input_shards > 1
|
194
|
+
many_to_one = output_shards > 1 && input_shards == 1
|
195
|
+
many_to_many = output_shards > 1 && input_shards > 1
|
196
|
+
|
197
|
+
connection_type = many_to_many ? RFlow::Configuration::BrokeredZMQConnection : RFlow::Configuration::ZMQConnection
|
198
|
+
|
199
|
+
conn = connection_type.create!(:name => spec[:name],
|
200
|
+
:output_port_key => spec[:output_port_key],
|
201
|
+
:input_port_key => spec[:input_port_key],
|
202
|
+
:output_port => output_port,
|
203
|
+
:input_port => input_port)
|
204
|
+
|
205
|
+
# bind on the cardinality-1 side, connect on the cardinality-n side
|
206
|
+
if in_shard_connection
|
207
|
+
conn.options['output_responsibility'] = 'connect'
|
208
|
+
conn.options['input_responsibility'] = 'bind'
|
209
|
+
conn.options['output_address'] = "inproc://rflow.#{conn.uuid}"
|
210
|
+
conn.options['input_address'] = "inproc://rflow.#{conn.uuid}"
|
211
|
+
elsif many_to_one
|
212
|
+
conn.options['output_responsibility'] = 'connect'
|
213
|
+
conn.options['input_responsibility'] = 'bind'
|
214
|
+
elsif one_to_many
|
215
|
+
conn.options['output_responsibility'] = 'bind'
|
216
|
+
conn.options['input_responsibility'] = 'connect'
|
217
|
+
end
|
218
|
+
|
219
|
+
conn.save!
|
220
|
+
conn
|
183
221
|
rescue Exception => e
|
184
222
|
# TODO: Figure out why an ArgumentError doesn't put the
|
185
223
|
# offending message into e.message, even though it is printed
|