rflow 1.0.0a4 → 1.0.0a5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -12
- data/lib/rflow/component/port.rb +20 -0
- data/lib/rflow/shard.rb +6 -0
- data/lib/rflow/version.rb +1 -1
- data/spec/rflow/component/port_spec.rb +16 -0
- data/spec/rflow/components/clock_spec.rb +12 -13
- data/spec/rflow/forward_to_input_port_spec.rb +8 -8
- data/spec/rflow/forward_to_output_port_spec.rb +4 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef1b7b3ae0fb683425fec5c44f0c1d39f7f29935
|
4
|
+
data.tar.gz: acc3b05b5b26d9286fe51c4c0d3e6a028d9c0f46
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d040a8d35aff247bd4200c2b5ac8f4ba4c9b8b9dc48a31292e68db1db07cc3e568a2a33067c600f54d99e3346bc60640f7338f395d9c9ddbc5caf7c7f725145d
|
7
|
+
data.tar.gz: 3f8b6b770e61ad2eb2ad17aa647d516d761d042c38491d958bbcb7b85f1b0d91fc80fa0cdd9d5a6e27af1308d05c2c3324c2e9a595bb1b076eedbb781c1c66a8
|
data/README.md
CHANGED
@@ -64,10 +64,11 @@ work. (You will probably get errors saying arcane things like
|
|
64
64
|
|
65
65
|
* __Connection__ - a directed link between an output port and an input
|
66
66
|
port. RFlow supports generalized connection types; however, only
|
67
|
-
ZeroMQ links are currently used.
|
67
|
+
ZeroMQ links are currently used. Round-robin and broadcast message
|
68
|
+
delivery are supported on a per-link basis.
|
68
69
|
|
69
70
|
* __Message__ - a bit of serialized data that is sent out an output
|
70
|
-
port and
|
71
|
+
port and received on an input port. Due to the serialization,
|
71
72
|
message types and schemas are explicitly defined. In a departure
|
72
73
|
from "pure" FBP, RFlow supports sending multiple message types via a
|
73
74
|
single connection.
|
@@ -228,7 +229,7 @@ The `provenance` is a way for a component to annotate a message with a
|
|
228
229
|
bit of data that should (by convention) be carried through the
|
229
230
|
workflow with the message, as well as being copied to derived
|
230
231
|
messages. For example, a TCP server component would spin up a TCP
|
231
|
-
server and, upon
|
232
|
+
server and, upon receiving a connection and packets on a session, it
|
232
233
|
would marshal the packets into `RFlow::Messsage`s and send them out
|
233
234
|
its output ports. Messages received on its input port, however, need
|
234
235
|
to have a way to be matched to the corresponding underlying TCP
|
@@ -237,7 +238,6 @@ component to add a bit of metadata (namely an identifier for the TCP
|
|
237
238
|
connection) such that later messages that contain the same provenance
|
238
239
|
can be matched to the correct underlying TCP connection.
|
239
240
|
|
240
|
-
|
241
241
|
The other parts of the message envelope are related to the embedded
|
242
242
|
data object. In addition to the data object itself (which is encoded
|
243
243
|
with a specific Avro schema), there are a few fields that describe the
|
@@ -319,7 +319,6 @@ message.data.int = 1024
|
|
319
319
|
messaga.data.default? # => false
|
320
320
|
```
|
321
321
|
|
322
|
-
|
323
322
|
## RFlow Workflow Configuration
|
324
323
|
|
325
324
|
RFlow currently stores its configuration in a SQLite database which
|
@@ -351,12 +350,13 @@ identify specific components.
|
|
351
350
|
either a `RFlow::Configuration::InputPort` or
|
352
351
|
`RFlow::Configuration::OutputPort`.
|
353
352
|
|
354
|
-
* connections - a connection between two ports via
|
353
|
+
* connections - a connection between two ports via foreign keys
|
355
354
|
`input_port_uuid` and `output_port_uuid`. Like ports, connections
|
356
355
|
are typed via AR STI (`RFlow::Configuration::ZMQConnection` and
|
357
|
-
|
356
|
+
`RFlow::Configuration::BrokeredZMQConnection` are the only
|
358
357
|
supported values for now) and have a YAML serialized `options`
|
359
|
-
hash
|
358
|
+
hash and a `delivery` type (`round-robin` or `broadcast`).
|
359
|
+
A connection also (potentially) defines the port keys.
|
360
360
|
|
361
361
|
RFlow also provides a RubyDSL for configuration-like file to be used
|
362
362
|
to load the database:
|
@@ -411,9 +411,10 @@ ZeroMQ communication between components in the same shard uses ZeroMQ's
|
|
411
411
|
components in different shards is accomplished with a ZeroMQ `ipc` socket.
|
412
412
|
In the case of a many-to-many connection (many workers in a producing
|
413
413
|
shard and many workers in a consuming shard), a ZeroMQ message broker
|
414
|
-
process is created to route the messages appropriately.
|
415
|
-
|
416
|
-
|
414
|
+
process is created to route the messages appropriately. By default,
|
415
|
+
senders round-robin to receivers, though broadcast delivery can be chosen
|
416
|
+
instead. Receivers fair-queue the messages from senders. Load balancing
|
417
|
+
based on receiver responsiveness is not currently implemented.
|
417
418
|
|
418
419
|
To define a custom shard in the Ruby DSL, use the `shard` method. For
|
419
420
|
example:
|
@@ -462,7 +463,9 @@ RFlow::Configuration::RubyDSL.configure do |config|
|
|
462
463
|
config.connect 'generate_ints1#out' => 'filter#in'
|
463
464
|
config.connect 'generate_ints2#out' => 'filter#in'
|
464
465
|
config.connect 'filter#filtered' => 'replicate#in'
|
465
|
-
|
466
|
+
# choosing broadcast delivery delivers a copy to each worker for
|
467
|
+
# the shard
|
468
|
+
config.connect 'filter#out' => 'output1#in', :delivery => 'broadcast'
|
466
469
|
config.connect 'filter#filtered' => 'output2#in'
|
467
470
|
end
|
468
471
|
```
|
data/lib/rflow/component/port.rb
CHANGED
@@ -83,6 +83,26 @@ class RFlow
|
|
83
83
|
connections_for[key] << connection
|
84
84
|
end
|
85
85
|
|
86
|
+
# Removes a connection from a given key
|
87
|
+
def remove_connection(key, connection)
|
88
|
+
RFlow.logger.debug "Removing #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) from port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
|
89
|
+
connections_for[key].delete(connection)
|
90
|
+
end
|
91
|
+
|
92
|
+
def collect_messages(key, receiver)
|
93
|
+
begin
|
94
|
+
connection = RFlow::MessageCollectingConnection.new.tap do |c|
|
95
|
+
c.messages = receiver
|
96
|
+
end
|
97
|
+
add_connection key, connection
|
98
|
+
|
99
|
+
yield if block_given?
|
100
|
+
connection
|
101
|
+
ensure
|
102
|
+
remove_connection key, connection if connection && block_given?
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
86
106
|
def direct_connect(other_port)
|
87
107
|
case other_port
|
88
108
|
when InputPort; add_connection nil, ForwardToInputPort.new(other_port)
|
data/lib/rflow/shard.rb
CHANGED
@@ -31,6 +31,7 @@ class RFlow
|
|
31
31
|
run_components!
|
32
32
|
rescue Exception => e
|
33
33
|
RFlow.logger.error "Error in worker, shutting down: #{e.class.name}: #{e.message}, because: #{e.backtrace.inspect}"
|
34
|
+
exit! 1
|
34
35
|
end
|
35
36
|
end
|
36
37
|
|
@@ -68,6 +69,11 @@ class RFlow
|
|
68
69
|
end
|
69
70
|
|
70
71
|
def shutdown!(signal)
|
72
|
+
RFlow.logger.debug "Shutting down components"
|
73
|
+
@components.each do |component|
|
74
|
+
RFlow.logger.debug "Shutting down component '#{component.name}' (#{component.uuid})"
|
75
|
+
component.shutdown!
|
76
|
+
end
|
71
77
|
EM.stop_event_loop
|
72
78
|
super
|
73
79
|
end
|
data/lib/rflow/version.rb
CHANGED
@@ -31,6 +31,22 @@ class RFlow
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
end
|
34
|
+
|
35
|
+
context "#(add|remove)_connection" do
|
36
|
+
it "should remove the connection" do
|
37
|
+
connection = double('connection')
|
38
|
+
allow(connection).to receive(:name)
|
39
|
+
allow(connection).to receive(:uuid)
|
40
|
+
allow(connection).to receive(:input_port_key)
|
41
|
+
|
42
|
+
described_class.new(nil).tap do |port|
|
43
|
+
port.add_connection(nil, connection)
|
44
|
+
expect(port[nil]).to include connection
|
45
|
+
port.remove_connection(nil, connection)
|
46
|
+
expect(port[nil]).not_to include connection
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
34
50
|
end
|
35
51
|
|
36
52
|
describe OutputPort do
|
@@ -7,18 +7,15 @@ class RFlow
|
|
7
7
|
ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
|
8
8
|
Configuration.migrate_database
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
|
+
let(:messages) { [] }
|
11
12
|
|
12
13
|
def clock(args = {})
|
13
14
|
Clock.new.tap do |c|
|
14
15
|
c.configure! args
|
15
|
-
c.tick_port.connect!
|
16
|
-
c.tick_port.add_connection nil, message_connection
|
17
16
|
end
|
18
17
|
end
|
19
18
|
|
20
|
-
def messages; message_connection.messages; end
|
21
|
-
|
22
19
|
it 'defaults configuration nicely' do
|
23
20
|
clock.tap do |c|
|
24
21
|
expect(c.clock_name).to eq('Clock')
|
@@ -51,14 +48,16 @@ class RFlow
|
|
51
48
|
|
52
49
|
it 'should generate a tick message when asked' do
|
53
50
|
clock.tap do |c|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
51
|
+
c.tick_port.collect_messages(nil, messages) do
|
52
|
+
now = Integer(Time.now.to_f * 1000)
|
53
|
+
expect(messages).to be_empty
|
54
|
+
c.tick
|
55
|
+
expect(messages).to have(1).message
|
56
|
+
messages.first.tap do |m|
|
57
|
+
expect(m.data_type_name).to eq('RFlow::Message::Clock::Tick')
|
58
|
+
expect(m.data.name).to eq('Clock')
|
59
|
+
expect(m.data.timestamp).to be >= now
|
60
|
+
end
|
62
61
|
end
|
63
62
|
end
|
64
63
|
end
|
@@ -7,8 +7,8 @@ class RFlow
|
|
7
7
|
Configuration.migrate_database
|
8
8
|
end
|
9
9
|
|
10
|
-
let(:
|
11
|
-
let(:
|
10
|
+
let(:filtered_messages) { [] }
|
11
|
+
let(:dropped_messages) { [] }
|
12
12
|
|
13
13
|
let(:generator) do
|
14
14
|
RFlow::Components::GenerateIntegerSequence.new.tap do |c|
|
@@ -20,16 +20,16 @@ class RFlow
|
|
20
20
|
let(:ruby_proc_filter) do
|
21
21
|
RFlow::Components::RubyProcFilter.new.tap do |c|
|
22
22
|
c.configure! 'filter_proc_string' => 'message.data.data_object % 2 == 0'
|
23
|
-
c.filtered.add_connection nil, filtered_message_connection
|
24
|
-
c.dropped.add_connection nil, dropped_message_connection
|
25
23
|
end
|
26
24
|
end
|
27
25
|
|
28
|
-
def filtered_messages; filtered_message_connection.messages; end
|
29
|
-
def dropped_messages; dropped_message_connection.messages; end
|
30
|
-
|
31
26
|
it 'should forward generated integers to be filtered by the proc filter' do
|
32
|
-
|
27
|
+
ruby_proc_filter.filtered.collect_messages(nil, filtered_messages) do
|
28
|
+
ruby_proc_filter.dropped.collect_messages(nil, dropped_messages) do
|
29
|
+
5.times { generator.generate }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
33
|
expect(filtered_messages).to have(3).messages
|
34
34
|
expect(filtered_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
|
35
35
|
expect(dropped_messages).to have(2).messages
|
@@ -7,7 +7,7 @@ class RFlow
|
|
7
7
|
Configuration.migrate_database
|
8
8
|
end
|
9
9
|
|
10
|
-
let(:
|
10
|
+
let(:messages) { [] }
|
11
11
|
|
12
12
|
let(:generator) do
|
13
13
|
RFlow::Components::GenerateIntegerSequence.new.tap do |c|
|
@@ -19,14 +19,13 @@ class RFlow
|
|
19
19
|
let(:ruby_proc_filter) do
|
20
20
|
RFlow::Components::RubyProcFilter.new.tap do |c|
|
21
21
|
c.configure! 'filter_proc_string' => 'message % 2 == 0'
|
22
|
-
c.filtered.add_connection nil, message_connection
|
23
22
|
end
|
24
23
|
end
|
25
24
|
|
26
|
-
def messages; message_connection.messages; end
|
27
|
-
|
28
25
|
it 'should place the messages on the output port, regardless of the filter' do
|
29
|
-
|
26
|
+
ruby_proc_filter.filtered.collect_messages(nil, messages) do
|
27
|
+
5.times { generator.generate }
|
28
|
+
end
|
30
29
|
expect(messages.map(&:data).map(&:data_object)).to eq([0, 1, 2, 3, 4])
|
31
30
|
end
|
32
31
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.0a5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael L. Artz
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-06-
|
11
|
+
date: 2014-06-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: uuidtools
|