rflow 1.0.0a2 → 1.0.0a3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/README.VAGRANT +63 -0
- data/README.md +118 -33
- data/Vagrantfile +53 -0
- data/bin/rflow +6 -1
- data/example/basic_extensions.rb +7 -8
- data/example/http_extensions.rb +7 -8
- data/lib/rflow/broker.rb +18 -0
- data/lib/rflow/child_process.rb +3 -1
- data/lib/rflow/component.rb +51 -61
- data/lib/rflow/component/port.rb +24 -15
- data/lib/rflow/configuration.rb +1 -0
- data/lib/rflow/configuration/connection.rb +35 -17
- data/lib/rflow/configuration/ruby_dsl.rb +47 -9
- data/lib/rflow/connection.rb +13 -9
- data/lib/rflow/connections/zmq_connection.rb +46 -3
- data/lib/rflow/daemon_process.rb +1 -1
- data/lib/rflow/master.rb +8 -1
- data/lib/rflow/shard.rb +8 -2
- data/lib/rflow/version.rb +1 -1
- data/rflow.gemspec +6 -6
- data/spec/fixtures/extensions_ints.rb +7 -8
- data/spec/rflow/component/port_spec.rb +16 -22
- data/spec/rflow/components/clock_spec.rb +12 -17
- data/spec/rflow/configuration/ruby_dsl_spec.rb +234 -46
- data/spec/rflow/configuration_spec.rb +5 -5
- data/spec/rflow/forward_to_input_port_spec.rb +10 -18
- data/spec/rflow/forward_to_output_port_spec.rb +6 -13
- data/spec/rflow/logger_spec.rb +6 -6
- data/spec/rflow/message/data/raw_spec.rb +3 -3
- data/spec/rflow/message_spec.rb +16 -16
- data/spec/rflow_spec.rb +37 -37
- data/spec/spec_helper.rb +3 -5
- metadata +20 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82d81c66d6a26aa814d88893e294951247434585
|
4
|
+
data.tar.gz: 03edc9684d65ffb4e0729000f8948f8dbe69c16c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7ca15c17aa77a9e15d5309c5e9dd22b533b0108ca6f8c29031899f8c824f3c4939c5c37e57bbe51db5834eefc40f0f52a840874af9660fbbe07729b8a66ea3d0
|
7
|
+
data.tar.gz: a8f7a76ca8eba6aec45cdf56433792938610e11332dd0523917795dd4a59edc84660d80b08a48a67c222f3a312ba6837b238a36e9db9e982b62fac2742f93417
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
ruby-2.1.
|
1
|
+
ruby-2.1.2
|
data/README.VAGRANT
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
How to build the vagrant boxes (VMWare Fusion 6):
|
2
|
+
|
3
|
+
First:
|
4
|
+
Add /Applications/VMware Fusion.app//Contents/Library/ to your PATH.
|
5
|
+
|
6
|
+
CentOS 6.2 (http://cbednarski.com/articles/creating-vagrant-base-box-for-centos-62/)
|
7
|
+
|
8
|
+
curl -O http://mirrors.usc.edu/pub/linux/distributions/centos/6.2/isos/x86_64/CentOS-6.2-x86_64-minimal.iso
|
9
|
+
Create a new VMWare box with the iso. Start the image. Don't use Easy Install.
|
10
|
+
Hostname: centos62
|
11
|
+
Root password: vagrant
|
12
|
+
Disk partition: Replace Existing Linux Partition.
|
13
|
+
Reboot.
|
14
|
+
Log in as root/vagrant.
|
15
|
+
vi /etc/sysconfig/network-scripts/ifcfg-eth0
|
16
|
+
Change ONBOOT="no" to yes.
|
17
|
+
Remove HWADDR line.
|
18
|
+
Add BOOTPROTO="dhcp"
|
19
|
+
vi /etc/udev/rules.d/70-persistent-net.rules
|
20
|
+
Replace the eth0 line with: SUBSYSTEM=="net", ACTION=="add", DRIVERS="?*", ATTR{type}=="1", KERNEL=="eth*", NAME="eth0"
|
21
|
+
shutdown -r now
|
22
|
+
|
23
|
+
Log back in as root.
|
24
|
+
VMWare Fusion > Virtual Machine > Install VMWare Tools
|
25
|
+
mkdir /media/cdrom
|
26
|
+
mount /dev/cdrom /media/cdrom
|
27
|
+
cd /tmp
|
28
|
+
tar -xzf /media/cdrom/VM[tab].tar.gz
|
29
|
+
yum install -y perl eject
|
30
|
+
/tmp/vmware-tools-distrib/vmware-install.pl --default
|
31
|
+
|
32
|
+
yum install -y sudo
|
33
|
+
useradd -m vagrant
|
34
|
+
usermod -aG wheel vagrant
|
35
|
+
echo vagrant | passwd vagrant --stdin
|
36
|
+
echo "vagrant ALL=(ALL) ALL" >> /etc/sudoers
|
37
|
+
echo "%wheel ALL=NOPASSWD: ALL" >> /etc/sudoers
|
38
|
+
echo 'Defaults env_keep="SSH_AUTH_SOCK"' >> /etc/sudoers
|
39
|
+
|
40
|
+
vi /etc/sudoers
|
41
|
+
Change requiretty to !requiretty
|
42
|
+
|
43
|
+
yum install -y openssh-server
|
44
|
+
echo "UseDNS no" >> /etc/ssh/sshd_config
|
45
|
+
|
46
|
+
mkdir -m 0700 /home/vagrant/.ssh
|
47
|
+
curl -s https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub > /home/vagrant/.ssh/authorized_keys
|
48
|
+
chown -R vagrant:vagrant /home/vagrant/.ssh
|
49
|
+
chmod -R 0600 /home/vagrant/.ssh/*
|
50
|
+
shutdown -r now
|
51
|
+
|
52
|
+
Verify logging in as vagrant/vagrant and `sudo ls /root` with no password.
|
53
|
+
|
54
|
+
sudo shutdown -h now
|
55
|
+
|
56
|
+
cd to wherever you have stored the VMWare box (~/Documents/Virtual Machines.localized/ by default).
|
57
|
+
vmware-vdiskmanager -d Virtual\ Disk.vmdk (ignore the warning message)
|
58
|
+
vmware-vdiskmanager -k Virtual\ Disk.vmdk (ignore the warning message)
|
59
|
+
cat > metadata.json
|
60
|
+
{
|
61
|
+
"provider":"vmware_fusion"
|
62
|
+
}
|
63
|
+
|
data/README.md
CHANGED
@@ -20,21 +20,34 @@ however only two are in current use, namely ZeroMQ connections and
|
|
20
20
|
Avro serialization.
|
21
21
|
|
22
22
|
RFlow currently runs as a single-threaded, evented system on top of
|
23
|
-
[
|
23
|
+
[EventMachine](http://rubyeventmachine.com/), meaning that any code
|
24
24
|
should be coded in an asynchronous style so as to not block the
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
EventMachine reactor (and thus block all the other components). Use
|
26
|
+
`EM.defer` and other such patterns, along with EventMachine plugins
|
27
|
+
for various servers and clients, to work in this style and defer
|
28
|
+
computation to background threads.
|
29
|
+
|
30
|
+
RFlow component workflows may be split into `shards` to improve
|
31
|
+
parallelism. Each shard is currently represented by a separate process,
|
32
|
+
though threads may be supported in the future. Multiple copies of a
|
33
|
+
shard may be instantiated, which will cooperate to round-robin
|
34
|
+
incoming messages.
|
28
35
|
|
29
36
|
Some of the long-term goals of RFlow are to allow for components and
|
30
37
|
portions of the workflow to be defined in any language that supports
|
31
|
-
Avro and ZeroMQ, which are numerous.
|
38
|
+
Avro and ZeroMQ, which are numerous. For this reason, the official
|
39
|
+
specification of an RFlow workflow is a SQLite database containing
|
40
|
+
information on its components, connections, ports, settings, etc.
|
41
|
+
There is a Ruby DSL that aids in populating the database but the intent
|
42
|
+
is that multiple processes and languages could access and manipulate
|
43
|
+
the database form.
|
32
44
|
|
33
45
|
## Developer Notes
|
34
46
|
|
35
47
|
You will need ZeroMQ preinstalled. Currently, EventMachine only supports
|
36
48
|
v3.2.4, not v4.x, so install that version. Older versions like 2.2 will not
|
37
|
-
work.
|
49
|
+
work. (You will probably get errors saying arcane things like
|
50
|
+
`assertion failed, mailbox.cpp(84)`).
|
38
51
|
|
39
52
|
## Definitions
|
40
53
|
|
@@ -50,8 +63,8 @@ work.
|
|
50
63
|
accessed by an array.
|
51
64
|
|
52
65
|
* __Connection__ - a directed link between an output port and an input
|
53
|
-
port. RFlow supports generalized connection types
|
54
|
-
ZeroMQ
|
66
|
+
port. RFlow supports generalized connection types; however, only
|
67
|
+
ZeroMQ links are currently used.
|
55
68
|
|
56
69
|
* __Message__ - a bit of serialized data that is sent out an output
|
57
70
|
port and recieved on an input port. Due to the serialization,
|
@@ -63,7 +76,6 @@ work.
|
|
63
76
|
components (nodes) are wired together via connections to their
|
64
77
|
respective output/input ports.
|
65
78
|
|
66
|
-
|
67
79
|
## Component Examples
|
68
80
|
|
69
81
|
The following describes the API of an RFlow component:
|
@@ -90,7 +102,7 @@ end
|
|
90
102
|
* `configure!` (called with a hash configuration) is called after the
|
91
103
|
component is instantiated but before the workflow has been wired or
|
92
104
|
any messages have been sent. Note that this is called outside the
|
93
|
-
|
105
|
+
EventMachine reactor.
|
94
106
|
|
95
107
|
* `run!` is called after all the components have been wired together
|
96
108
|
with connections and the entire workflow has been created. For a
|
@@ -98,11 +110,11 @@ end
|
|
98
110
|
be sent. For example, if the component is reading from a file, this
|
99
111
|
is where the file will be opened, the contents read into a message,
|
100
112
|
and the message sent out the output port. `run!` is called within
|
101
|
-
the
|
113
|
+
the EventMachine reactor.
|
102
114
|
|
103
115
|
* `process_message` is an evented callback that is called whenever the
|
104
116
|
component receives a message on one of its input ports.
|
105
|
-
`process_message` is called
|
117
|
+
`process_message` is called within the EventMachine reactor
|
106
118
|
|
107
119
|
* `shutdown!` is called when the flow is being terminated, and is
|
108
120
|
meant to allow the components to do penultimate processing and send
|
@@ -186,20 +198,19 @@ configuration):
|
|
186
198
|
class RFlow::Components::FileOutput < RFlow::Component
|
187
199
|
input_port :in
|
188
200
|
|
189
|
-
attr_accessor :output_file_path
|
201
|
+
attr_accessor :output_file_path
|
190
202
|
|
191
203
|
def configure!(config)
|
192
204
|
self.output_file_path = config['output_file_path']
|
193
|
-
self.output_file = File.new output_file_path, 'w+'
|
194
205
|
end
|
195
206
|
|
196
207
|
def process_message(input_port, input_port_key, connection, message)
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
208
|
+
File.open(output_file_path, 'a') do |f|
|
209
|
+
f.flock(File::LOCK_EX)
|
210
|
+
f.puts message.data.data_object.inspect
|
211
|
+
f.flush
|
212
|
+
f.flock(File::LOCK_UN)
|
213
|
+
end
|
203
214
|
end
|
204
215
|
end
|
205
216
|
```
|
@@ -314,7 +325,7 @@ messaga.data.default? # => false
|
|
314
325
|
RFlow currently stores its configuration in a SQLite database which
|
315
326
|
are internally accessed via ActiveRecord. Given that SQLite is a
|
316
327
|
rather simple and standard interface, non-RFlow components could
|
317
|
-
access it and determine their
|
328
|
+
access it and determine their respective ZMQ connections.
|
318
329
|
|
319
330
|
DB schemas for the configuration database are in
|
320
331
|
[lib/rflow/configuration/migrations](lib/rflow/configuration/migrations)
|
@@ -323,26 +334,29 @@ tables uses a UUID primary key, and UUIDs are used within RFlow to
|
|
323
334
|
identify specific components.
|
324
335
|
|
325
336
|
* settings - general application settings, such as log levels, app
|
326
|
-
names, directories, etc
|
337
|
+
names, directories, etc.
|
338
|
+
|
339
|
+
* shards - a list of the shards defined for the workflow, including
|
340
|
+
UUID, type, and number of workers for the shard
|
327
341
|
|
328
342
|
* components - a list of the components including its name,
|
329
|
-
specification (Ruby class), and options. Note that the options are
|
343
|
+
specification (Ruby class), shard, and options. Note that the options are
|
330
344
|
serialized to the database as YAML, and components should understand
|
331
345
|
that the round-trip through the database might not be perfect (e.g.
|
332
346
|
Ruby symbols might become strings). A component also has a number of
|
333
347
|
input ports and output ports.
|
334
348
|
|
335
349
|
* ports - belonging to a component (via `component_uuid` foreign key),
|
336
|
-
also has a `type`
|
350
|
+
also has a `type` column for ActiveRecord STI, which gets set to
|
337
351
|
either a `RFlow::Configuration::InputPort` or
|
338
352
|
`RFlow::Configuration::OutputPort`.
|
339
353
|
|
340
354
|
* connections - a connection between two ports via foriegn keys
|
341
355
|
`input_port_uuid` and `output_port_uuid`. Like ports, connections
|
342
|
-
are typed via AR STI (`RFlow::Configuration::ZMQConnection`
|
343
|
-
|
344
|
-
|
345
|
-
keys.
|
356
|
+
are typed via AR STI (`RFlow::Configuration::ZMQConnection` and
|
357
|
+
'RFlow::Configuration::BrokeredZMGConnection` are the only
|
358
|
+
supported values for now) and have a YAML serialized `options`
|
359
|
+
hash. A connection also (potentially) defines the port keys.
|
346
360
|
|
347
361
|
RFlow also provides a RubyDSL for configuration-like file to be used
|
348
362
|
to load the database:
|
@@ -351,10 +365,9 @@ to load the database:
|
|
351
365
|
RFlow::Configuration::RubyDSL.configure do |config|
|
352
366
|
# Configure the settings, which include paths for various files, log
|
353
367
|
# levels, and component specific stuffs
|
354
|
-
config.setting
|
355
|
-
config.setting
|
356
|
-
|
357
|
-
config.setting('rflow.application_name', 'testapp')
|
368
|
+
config.setting 'rflow.log_level', 'DEBUG'
|
369
|
+
config.setting 'rflow.application_directory_path', '../tmp'
|
370
|
+
config.setting 'rflow.application_name', 'testapp'
|
358
371
|
|
359
372
|
# Instantiate components
|
360
373
|
config.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', {
|
@@ -386,10 +399,82 @@ RFlow::Configuration::RubyDSL.configure do |config|
|
|
386
399
|
end
|
387
400
|
```
|
388
401
|
|
402
|
+
## Parallelism
|
403
|
+
|
404
|
+
RFlow supports parallelizing workflows and splitting them into multiple
|
405
|
+
`shard`s. By default, components declared in the Ruby DSL exist in the
|
406
|
+
default shard, named `DEFAULT`. There is only one worker for the default
|
407
|
+
shard.
|
408
|
+
|
409
|
+
ZeroMQ communication between components in the same shard uses ZeroMQ's
|
410
|
+
`inproc` socket type for maximum performance. ZeroMQ communication between
|
411
|
+
components in different shards is accomplished with a ZeroMQ `ipc` socket.
|
412
|
+
In the case of a many-to-many connection (many workers in a producing
|
413
|
+
shard and many workers in a consuming shard), a ZeroMQ message broker
|
414
|
+
process is created to route the messages appropriately. Senders round-robin
|
415
|
+
to receivers and receivers fair-queue the messages from the senders.
|
416
|
+
Load balancing based on receiver responsiveness is not currently implemented.
|
417
|
+
|
418
|
+
To define a custom shard in the Ruby DSL, use the `shard` method. For
|
419
|
+
example:
|
420
|
+
|
421
|
+
```ruby
|
422
|
+
RFlow::Configuration::RubyDSL.configure do |config|
|
423
|
+
# Configure the settings, which include paths for various files, log
|
424
|
+
# levels, and component specific stuffs
|
425
|
+
config.setting 'rflow.log_level', 'DEBUG'
|
426
|
+
config.setting 'rflow.application_directory_path', '../tmp'
|
427
|
+
config.setting 'rflow.application_name', 'testapp'
|
428
|
+
|
429
|
+
config.shard 'integers', :process => 2 do |shard|
|
430
|
+
shard.component 'generate_ints1', 'RFlow::Components::GenerateIntegerSequence', {
|
431
|
+
'start' => 0,
|
432
|
+
'finish' => 10,
|
433
|
+
'step' => 3,
|
434
|
+
'interval_seconds' => 1
|
435
|
+
}
|
436
|
+
shard.component 'generate_ints2', 'RFlow::Components::GenerateIntegerSequence', {
|
437
|
+
'start' => 20,
|
438
|
+
'finish' => 30
|
439
|
+
}
|
440
|
+
end
|
441
|
+
|
442
|
+
# another style of specifying type and count; count defaults to 1
|
443
|
+
config.shard 'filters', :type => :process, :count => 1 do |shard|
|
444
|
+
shard.component 'filter', 'RFlow::Components::RubyProcFilter', {
|
445
|
+
'filter_proc_string' => 'lambda {|message| true}'
|
446
|
+
}
|
447
|
+
end
|
448
|
+
|
449
|
+
# another way of specifying type
|
450
|
+
config.process 'filters', :count => 2 do |shard|
|
451
|
+
shard.component 'output1', 'RFlow::Components::FileOutput', {
|
452
|
+
'output_file_path' => '/tmp/out1'
|
453
|
+
}
|
454
|
+
end
|
455
|
+
|
456
|
+
# this component will be created in the DEFAULT shard
|
457
|
+
config.component 'output2', 'RFlow::Components::FileOutput', {
|
458
|
+
'output_file_path' => '/tmp/out2'
|
459
|
+
}
|
460
|
+
|
461
|
+
# Wire components together
|
462
|
+
config.connect 'generate_ints1#out' => 'filter#in'
|
463
|
+
config.connect 'generate_ints2#out' => 'filter#in'
|
464
|
+
config.connect 'filter#filtered' => 'replicate#in'
|
465
|
+
config.connect 'filter#out' => 'output1#in'
|
466
|
+
config.connect 'filter#filtered' => 'output2#in'
|
467
|
+
end
|
468
|
+
```
|
469
|
+
|
470
|
+
At runtime, shards with no components defined will have no workers and
|
471
|
+
will not be started. (So, if you put all components in a custom shard,
|
472
|
+
no `DEFAULT` workers will be seen.)
|
473
|
+
|
389
474
|
## Command-Line Operation
|
390
475
|
|
391
476
|
RFlow includes the `rflow` binary that can load a database from a Ruby
|
392
|
-
DSL, as well as start/stop the
|
477
|
+
DSL, as well as start/stop the workflow application as a daemon.
|
393
478
|
Invoking the `rflow` binary without any options will give a brief help:
|
394
479
|
|
395
480
|
```
|
data/Vagrantfile
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# -*- mode: ruby -*-
|
2
|
+
# vi: set ft=ruby :
|
3
|
+
VAGRANTFILE_API_VERSION = "2"
|
4
|
+
|
5
|
+
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
6
|
+
config.vm.define 'centos62' do |c|
|
7
|
+
c.vm.box = "jstoneham/rflow-centos62"
|
8
|
+
end
|
9
|
+
config.vm.define 'centos64' do |c|
|
10
|
+
c.vm.box = "box-cutter/centos64"
|
11
|
+
end
|
12
|
+
config.vm.define 'centos65' do |c|
|
13
|
+
c.vm.box = "chef/centos-6.5"
|
14
|
+
end
|
15
|
+
|
16
|
+
config.vm.synced_folder '.', '/rflow'
|
17
|
+
# bring over rflow examples; use rsync so it's safe to create IPCs in the rflow-examples directory
|
18
|
+
# (that is, avoid NFS)
|
19
|
+
# run 'vagrant rsync-auto' to get syncing to happen automatically
|
20
|
+
config.vm.synced_folder '../rflow_examples', '/rflow_examples', type: 'rsync', rsync__exclude: '.git/'
|
21
|
+
config.vm.synced_folder '../rflow-components-http', '/rflow-components-http'
|
22
|
+
|
23
|
+
# forward http for rflow testing
|
24
|
+
config.vm.network "forwarded_port", guest: 8000, host: 8000
|
25
|
+
|
26
|
+
# install RPM dependencies for rflow and zeromq
|
27
|
+
config.vm.provision "shell", privileged: true, inline: <<-EOS
|
28
|
+
curl -O https://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
|
29
|
+
rpm -ivh epel-release-6-8.noarch.rpm
|
30
|
+
yum -y install libyaml-devel patch libffi-devel glibc-headers autoconf gcc-c++ glibc-devel readline-devel zlib-devel openssl-devel automake libtool bison git sqlite-devel rpm-build libuuid-devel vim
|
31
|
+
EOS
|
32
|
+
|
33
|
+
# build zeromq as vagrant user
|
34
|
+
config.vm.provision "shell", privileged: false, inline: <<-EOS
|
35
|
+
curl -O http://download.zeromq.org/zeromq-3.2.4.tar.gz
|
36
|
+
rpmbuild -tb zeromq-3.2.4.tar.gz
|
37
|
+
EOS
|
38
|
+
|
39
|
+
# install zeromq
|
40
|
+
config.vm.provision "shell", privileged: true, inline: <<-EOS
|
41
|
+
rpm -ivh ~vagrant/rpmbuild/RPMS/x86_64/zeromq-*
|
42
|
+
EOS
|
43
|
+
|
44
|
+
# set up RVM and bundler
|
45
|
+
config.vm.provision "shell", privileged: false, inline: <<-EOS
|
46
|
+
rm -f .profile
|
47
|
+
curl -sSL https://get.rvm.io | bash -s stable
|
48
|
+
source .rvm/scripts/rvm
|
49
|
+
rvm install `cat /rflow/.ruby-version`
|
50
|
+
cd /rflow
|
51
|
+
bundle update
|
52
|
+
EOS
|
53
|
+
end
|
data/bin/rflow
CHANGED
@@ -66,7 +66,12 @@ end
|
|
66
66
|
|
67
67
|
# Now require rflow because the following parts of the startup require
|
68
68
|
# pieces (usually RFlow::Configuration or RFlow.logger)
|
69
|
-
|
69
|
+
begin
|
70
|
+
require 'rflow'
|
71
|
+
rescue Exception => e
|
72
|
+
STDERR.puts "Error loading RFlow: #{e.class} - #{e.message}"
|
73
|
+
exit 1
|
74
|
+
end
|
70
75
|
|
71
76
|
# Set up the startup logging, which is distinct from the runtime
|
72
77
|
# logging that is defined in the config database. The startup logging
|
data/example/basic_extensions.rb
CHANGED
@@ -14,21 +14,20 @@ end
|
|
14
14
|
RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Integer', SimpleDataExtension)
|
15
15
|
|
16
16
|
class RFlow::Components::FileOutput < RFlow::Component
|
17
|
-
attr_accessor :output_file_path
|
17
|
+
attr_accessor :output_file_path
|
18
18
|
input_port :in
|
19
19
|
|
20
20
|
def configure!(config)
|
21
21
|
self.output_file_path = config['output_file_path']
|
22
|
-
self.output_file = File.new output_file_path, 'w+'
|
23
22
|
end
|
24
23
|
|
25
24
|
def process_message(input_port, input_port_key, connection, message)
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
25
|
+
File.open(output_file_path, 'a') do |f|
|
26
|
+
f.flock(File::LOCK_EX)
|
27
|
+
f.puts message.data.data_object.inspect
|
28
|
+
f.flush
|
29
|
+
f.flock(File::LOCK_UN)
|
30
|
+
end
|
32
31
|
end
|
33
32
|
end
|
34
33
|
|
data/example/http_extensions.rb
CHANGED
@@ -5,21 +5,20 @@ require 'eventmachine'
|
|
5
5
|
require 'evma_httpserver'
|
6
6
|
|
7
7
|
class RFlow::Components::FileOutput < RFlow::Component
|
8
|
-
attr_accessor :output_file_path
|
8
|
+
attr_accessor :output_file_path
|
9
9
|
input_port :in
|
10
10
|
|
11
11
|
def configure!(config)
|
12
12
|
self.output_file_path = config['output_file_path']
|
13
|
-
self.output_file = File.new output_file_path, 'w+'
|
14
13
|
end
|
15
14
|
|
16
15
|
def process_message(input_port, input_port_key, connection, message)
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
16
|
+
File.open(output_file_path, 'a') do |f|
|
17
|
+
f.flock(File::LOCK_EX)
|
18
|
+
f.puts message.data.data_object.inspect
|
19
|
+
f.flush
|
20
|
+
f.flock(File::LOCK_UN)
|
21
|
+
end
|
23
22
|
end
|
24
23
|
end
|
25
24
|
|