rflow 1.0.0a4 → 1.0.0a5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -12
- data/lib/rflow/component/port.rb +20 -0
- data/lib/rflow/shard.rb +6 -0
- data/lib/rflow/version.rb +1 -1
- data/spec/rflow/component/port_spec.rb +16 -0
- data/spec/rflow/components/clock_spec.rb +12 -13
- data/spec/rflow/forward_to_input_port_spec.rb +8 -8
- data/spec/rflow/forward_to_output_port_spec.rb +4 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef1b7b3ae0fb683425fec5c44f0c1d39f7f29935
|
4
|
+
data.tar.gz: acc3b05b5b26d9286fe51c4c0d3e6a028d9c0f46
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d040a8d35aff247bd4200c2b5ac8f4ba4c9b8b9dc48a31292e68db1db07cc3e568a2a33067c600f54d99e3346bc60640f7338f395d9c9ddbc5caf7c7f725145d
|
7
|
+
data.tar.gz: 3f8b6b770e61ad2eb2ad17aa647d516d761d042c38491d958bbcb7b85f1b0d91fc80fa0cdd9d5a6e27af1308d05c2c3324c2e9a595bb1b076eedbb781c1c66a8
|
data/README.md
CHANGED
@@ -64,10 +64,11 @@ work. (You will probably get errors saying arcane things like
|
|
64
64
|
|
65
65
|
* __Connection__ - a directed link between an output port and an input
|
66
66
|
port. RFlow supports generalized connection types; however, only
|
67
|
-
ZeroMQ links are currently used.
|
67
|
+
ZeroMQ links are currently used. Round-robin and broadcast message
|
68
|
+
delivery are supported on a per-link basis.
|
68
69
|
|
69
70
|
* __Message__ - a bit of serialized data that is sent out an output
|
70
|
-
port and
|
71
|
+
port and received on an input port. Due to the serialization,
|
71
72
|
message types and schemas are explicitly defined. In a departure
|
72
73
|
from "pure" FBP, RFlow supports sending multiple message types via a
|
73
74
|
single connection.
|
@@ -228,7 +229,7 @@ The `provenance` is a way for a component to annotate a message with a
|
|
228
229
|
bit of data that should (by convention) be carried through the
|
229
230
|
workflow with the message, as well as being copied to derived
|
230
231
|
messages. For example, a TCP server component would spin up a TCP
|
231
|
-
server and, upon
|
232
|
+
server and, upon receiving a connection and packets on a session, it
|
232
233
|
would marshal the packets into `RFlow::Messsage`s and send them out
|
233
234
|
its output ports. Messages received on its input port, however, need
|
234
235
|
to have a way to be matched to the corresponding underlying TCP
|
@@ -237,7 +238,6 @@ component to add a bit of metadata (namely an identifier for the TCP
|
|
237
238
|
connection) such that later messages that contain the same provenance
|
238
239
|
can be matched to the correct underlying TCP connection.
|
239
240
|
|
240
|
-
|
241
241
|
The other parts of the message envelope are related to the embedded
|
242
242
|
data object. In addition to the data object itself (which is encoded
|
243
243
|
with a specific Avro schema), there are a few fields that describe the
|
@@ -319,7 +319,6 @@ message.data.int = 1024
|
|
319
319
|
messaga.data.default? # => false
|
320
320
|
```
|
321
321
|
|
322
|
-
|
323
322
|
## RFlow Workflow Configuration
|
324
323
|
|
325
324
|
RFlow currently stores its configuration in a SQLite database which
|
@@ -351,12 +350,13 @@ identify specific components.
|
|
351
350
|
either a `RFlow::Configuration::InputPort` or
|
352
351
|
`RFlow::Configuration::OutputPort`.
|
353
352
|
|
354
|
-
* connections - a connection between two ports via
|
353
|
+
* connections - a connection between two ports via foreign keys
|
355
354
|
`input_port_uuid` and `output_port_uuid`. Like ports, connections
|
356
355
|
are typed via AR STI (`RFlow::Configuration::ZMQConnection` and
|
357
|
-
|
356
|
+
`RFlow::Configuration::BrokeredZMQConnection` are the only
|
358
357
|
supported values for now) and have a YAML serialized `options`
|
359
|
-
hash
|
358
|
+
hash and a `delivery` type (`round-robin` or `broadcast`).
|
359
|
+
A connection also (potentially) defines the port keys.
|
360
360
|
|
361
361
|
RFlow also provides a RubyDSL for configuration-like file to be used
|
362
362
|
to load the database:
|
@@ -411,9 +411,10 @@ ZeroMQ communication between components in the same shard uses ZeroMQ's
|
|
411
411
|
components in different shards is accomplished with a ZeroMQ `ipc` socket.
|
412
412
|
In the case of a many-to-many connection (many workers in a producing
|
413
413
|
shard and many workers in a consuming shard), a ZeroMQ message broker
|
414
|
-
process is created to route the messages appropriately.
|
415
|
-
|
416
|
-
|
414
|
+
process is created to route the messages appropriately. By default,
|
415
|
+
senders round-robin to receivers, though broadcast delivery can be chosen
|
416
|
+
instead. Receivers fair-queue the messages from senders. Load balancing
|
417
|
+
based on receiver responsiveness is not currently implemented.
|
417
418
|
|
418
419
|
To define a custom shard in the Ruby DSL, use the `shard` method. For
|
419
420
|
example:
|
@@ -462,7 +463,9 @@ RFlow::Configuration::RubyDSL.configure do |config|
|
|
462
463
|
config.connect 'generate_ints1#out' => 'filter#in'
|
463
464
|
config.connect 'generate_ints2#out' => 'filter#in'
|
464
465
|
config.connect 'filter#filtered' => 'replicate#in'
|
465
|
-
|
466
|
+
# choosing broadcast delivery delivers a copy to each worker for
|
467
|
+
# the shard
|
468
|
+
config.connect 'filter#out' => 'output1#in', :delivery => 'broadcast'
|
466
469
|
config.connect 'filter#filtered' => 'output2#in'
|
467
470
|
end
|
468
471
|
```
|
data/lib/rflow/component/port.rb
CHANGED
@@ -83,6 +83,26 @@ class RFlow
|
|
83
83
|
connections_for[key] << connection
|
84
84
|
end
|
85
85
|
|
86
|
+
# Removes a connection from a given key
|
87
|
+
def remove_connection(key, connection)
|
88
|
+
RFlow.logger.debug "Removing #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) from port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
|
89
|
+
connections_for[key].delete(connection)
|
90
|
+
end
|
91
|
+
|
92
|
+
def collect_messages(key, receiver)
|
93
|
+
begin
|
94
|
+
connection = RFlow::MessageCollectingConnection.new.tap do |c|
|
95
|
+
c.messages = receiver
|
96
|
+
end
|
97
|
+
add_connection key, connection
|
98
|
+
|
99
|
+
yield if block_given?
|
100
|
+
connection
|
101
|
+
ensure
|
102
|
+
remove_connection key, connection if connection && block_given?
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
86
106
|
def direct_connect(other_port)
|
87
107
|
case other_port
|
88
108
|
when InputPort; add_connection nil, ForwardToInputPort.new(other_port)
|
data/lib/rflow/shard.rb
CHANGED
@@ -31,6 +31,7 @@ class RFlow
|
|
31
31
|
run_components!
|
32
32
|
rescue Exception => e
|
33
33
|
RFlow.logger.error "Error in worker, shutting down: #{e.class.name}: #{e.message}, because: #{e.backtrace.inspect}"
|
34
|
+
exit! 1
|
34
35
|
end
|
35
36
|
end
|
36
37
|
|
@@ -68,6 +69,11 @@ class RFlow
|
|
68
69
|
end
|
69
70
|
|
70
71
|
def shutdown!(signal)
|
72
|
+
RFlow.logger.debug "Shutting down components"
|
73
|
+
@components.each do |component|
|
74
|
+
RFlow.logger.debug "Shutting down component '#{component.name}' (#{component.uuid})"
|
75
|
+
component.shutdown!
|
76
|
+
end
|
71
77
|
EM.stop_event_loop
|
72
78
|
super
|
73
79
|
end
|
data/lib/rflow/version.rb
CHANGED
@@ -31,6 +31,22 @@ class RFlow
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
end
|
34
|
+
|
35
|
+
context "#(add|remove)_connection" do
|
36
|
+
it "should remove the connection" do
|
37
|
+
connection = double('connection')
|
38
|
+
allow(connection).to receive(:name)
|
39
|
+
allow(connection).to receive(:uuid)
|
40
|
+
allow(connection).to receive(:input_port_key)
|
41
|
+
|
42
|
+
described_class.new(nil).tap do |port|
|
43
|
+
port.add_connection(nil, connection)
|
44
|
+
expect(port[nil]).to include connection
|
45
|
+
port.remove_connection(nil, connection)
|
46
|
+
expect(port[nil]).not_to include connection
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
34
50
|
end
|
35
51
|
|
36
52
|
describe OutputPort do
|
@@ -7,18 +7,15 @@ class RFlow
|
|
7
7
|
ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
|
8
8
|
Configuration.migrate_database
|
9
9
|
end
|
10
|
-
|
10
|
+
|
11
|
+
let(:messages) { [] }
|
11
12
|
|
12
13
|
def clock(args = {})
|
13
14
|
Clock.new.tap do |c|
|
14
15
|
c.configure! args
|
15
|
-
c.tick_port.connect!
|
16
|
-
c.tick_port.add_connection nil, message_connection
|
17
16
|
end
|
18
17
|
end
|
19
18
|
|
20
|
-
def messages; message_connection.messages; end
|
21
|
-
|
22
19
|
it 'defaults configuration nicely' do
|
23
20
|
clock.tap do |c|
|
24
21
|
expect(c.clock_name).to eq('Clock')
|
@@ -51,14 +48,16 @@ class RFlow
|
|
51
48
|
|
52
49
|
it 'should generate a tick message when asked' do
|
53
50
|
clock.tap do |c|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
51
|
+
c.tick_port.collect_messages(nil, messages) do
|
52
|
+
now = Integer(Time.now.to_f * 1000)
|
53
|
+
expect(messages).to be_empty
|
54
|
+
c.tick
|
55
|
+
expect(messages).to have(1).message
|
56
|
+
messages.first.tap do |m|
|
57
|
+
expect(m.data_type_name).to eq('RFlow::Message::Clock::Tick')
|
58
|
+
expect(m.data.name).to eq('Clock')
|
59
|
+
expect(m.data.timestamp).to be >= now
|
60
|
+
end
|
62
61
|
end
|
63
62
|
end
|
64
63
|
end
|
@@ -7,8 +7,8 @@ class RFlow
|
|
7
7
|
Configuration.migrate_database
|
8
8
|
end
|
9
9
|
|
10
|
-
let(:
|
11
|
-
let(:
|
10
|
+
let(:filtered_messages) { [] }
|
11
|
+
let(:dropped_messages) { [] }
|
12
12
|
|
13
13
|
let(:generator) do
|
14
14
|
RFlow::Components::GenerateIntegerSequence.new.tap do |c|
|
@@ -20,16 +20,16 @@ class RFlow
|
|
20
20
|
let(:ruby_proc_filter) do
|
21
21
|
RFlow::Components::RubyProcFilter.new.tap do |c|
|
22
22
|
c.configure! 'filter_proc_string' => 'message.data.data_object % 2 == 0'
|
23
|
-
c.filtered.add_connection nil, filtered_message_connection
|
24
|
-
c.dropped.add_connection nil, dropped_message_connection
|
25
23
|
end
|
26
24
|
end
|
27
25
|
|
28
|
-
def filtered_messages; filtered_message_connection.messages; end
|
29
|
-
def dropped_messages; dropped_message_connection.messages; end
|
30
|
-
|
31
26
|
it 'should forward generated integers to be filtered by the proc filter' do
|
32
|
-
|
27
|
+
ruby_proc_filter.filtered.collect_messages(nil, filtered_messages) do
|
28
|
+
ruby_proc_filter.dropped.collect_messages(nil, dropped_messages) do
|
29
|
+
5.times { generator.generate }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
33
|
expect(filtered_messages).to have(3).messages
|
34
34
|
expect(filtered_messages.map(&:data).map(&:data_object)).to eq([0, 2, 4])
|
35
35
|
expect(dropped_messages).to have(2).messages
|
@@ -7,7 +7,7 @@ class RFlow
|
|
7
7
|
Configuration.migrate_database
|
8
8
|
end
|
9
9
|
|
10
|
-
let(:
|
10
|
+
let(:messages) { [] }
|
11
11
|
|
12
12
|
let(:generator) do
|
13
13
|
RFlow::Components::GenerateIntegerSequence.new.tap do |c|
|
@@ -19,14 +19,13 @@ class RFlow
|
|
19
19
|
let(:ruby_proc_filter) do
|
20
20
|
RFlow::Components::RubyProcFilter.new.tap do |c|
|
21
21
|
c.configure! 'filter_proc_string' => 'message % 2 == 0'
|
22
|
-
c.filtered.add_connection nil, message_connection
|
23
22
|
end
|
24
23
|
end
|
25
24
|
|
26
|
-
def messages; message_connection.messages; end
|
27
|
-
|
28
25
|
it 'should place the messages on the output port, regardless of the filter' do
|
29
|
-
|
26
|
+
ruby_proc_filter.filtered.collect_messages(nil, messages) do
|
27
|
+
5.times { generator.generate }
|
28
|
+
end
|
30
29
|
expect(messages.map(&:data).map(&:data_object)).to eq([0, 1, 2, 3, 4])
|
31
30
|
end
|
32
31
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.0a5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael L. Artz
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-06-
|
11
|
+
date: 2014-06-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: uuidtools
|