deadly_serious 1.0.2 → 2.0.0.pre.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Guardfile +2 -2
- data/deadly_serious.gemspec +1 -0
- data/lib/deadly_serious.rb +7 -6
- data/lib/deadly_serious/engine/auto_pipe.rb +33 -21
- data/lib/deadly_serious/engine/channel.rb +15 -130
- data/lib/deadly_serious/engine/channel/file_channel.rb +50 -0
- data/lib/deadly_serious/engine/channel/pipe_channel.rb +59 -0
- data/lib/deadly_serious/engine/channel/socket/master_mind.rb +42 -0
- data/lib/deadly_serious/engine/channel/socket/minion.rb +24 -0
- data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb +34 -0
- data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb +26 -0
- data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb +32 -0
- data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb +30 -0
- data/lib/deadly_serious/engine/channel/socket_channel.rb +75 -0
- data/lib/deadly_serious/engine/commands.rb +29 -8
- data/lib/deadly_serious/engine/config.rb +55 -0
- data/lib/deadly_serious/engine/file_monitor.rb +57 -0
- data/lib/deadly_serious/engine/json_io.rb +13 -11
- data/lib/deadly_serious/engine/pipeline.rb +31 -87
- data/lib/deadly_serious/engine/ruby_object_container.rb +42 -0
- data/lib/deadly_serious/engine/so_command_container.rb +56 -0
- data/lib/deadly_serious/processes/converter.rb +12 -0
- data/lib/deadly_serious/processes/lambda.rb +4 -2
- data/lib/deadly_serious/processes/resilient_splitter.rb +1 -1
- data/lib/deadly_serious/version.rb +1 -1
- data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb +41 -0
- data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb +159 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/commands_spec.rb +0 -0
- data/spec/lib/deadly_serious/engine/file_monitor_spec.rb +69 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/json_io_spec.rb +0 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/pipeline_spec.rb +37 -40
- data/spec/spec_helper.rb +4 -1
- metadata +51 -14
- data/lib/deadly_serious/engine/lazy_io.rb +0 -82
- data/lib/deadly_serious/engine/open_io.rb +0 -39
- data/lib/deadly_serious/processes/joiner.rb +0 -15
@@ -3,28 +3,27 @@ module DeadlySerious
|
|
3
3
|
class Pipeline
|
4
4
|
include DeadlySerious::Engine::Commands
|
5
5
|
|
6
|
-
attr_reader :
|
6
|
+
attr_reader :pids, :config
|
7
7
|
|
8
8
|
def initialize(data_dir: './data',
|
9
9
|
pipe_dir: "/tmp/deadly_serious/#{Process.pid}",
|
10
10
|
preserve_pipe_dir: false,
|
11
11
|
&block)
|
12
|
-
|
13
|
-
@
|
12
|
+
|
13
|
+
@config = Config.new(data_dir: data_dir, pipe_dir: pipe_dir, preserve_pipe_dir: preserve_pipe_dir)
|
14
14
|
@block = block
|
15
15
|
@pids = []
|
16
|
-
Channel.config(data_dir, pipe_dir, preserve_pipe_dir)
|
17
16
|
end
|
18
17
|
|
19
18
|
def run
|
20
|
-
|
19
|
+
@config.setup
|
21
20
|
@block.call(self)
|
22
21
|
wait_children
|
23
22
|
rescue => e
|
24
23
|
kill_children
|
25
24
|
raise e
|
26
25
|
ensure
|
27
|
-
|
26
|
+
@config.teardown if @config
|
28
27
|
end
|
29
28
|
|
30
29
|
# Wait all sub processes to finish before
|
@@ -43,114 +42,59 @@ module DeadlySerious
|
|
43
42
|
# prefer the simpler {DeadlySerious::Engine::Commands#spawn_class} or
|
44
43
|
# the {DeadlySerious::Engine::Commands#spawn} methods.
|
45
44
|
def spawn_process(class_or_object, *args, process_name: nil, readers: [last_pipe], writers: [next_pipe])
|
46
|
-
|
47
|
-
# TODO if we have no readers, and this is the first process, read from STDIN
|
48
|
-
# TODO if we have no writers, alarm! (how about data sinks???)
|
49
|
-
# TODO if we have no writers, and this is the last process, write to STDOUT
|
50
|
-
process_name ||= class_or_object.respond_to?(:name) ? class_or_object.name : class_or_object.to_s
|
51
|
-
writers.each { |writer| create_pipe(writer) }
|
45
|
+
writers.compact.each { |w| Channel.of_type(w).create(w, @config) }
|
52
46
|
@pids << fork do
|
53
47
|
begin
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
48
|
+
container = RubyObjectContainer.new(class_or_object,
|
49
|
+
args,
|
50
|
+
process_name,
|
51
|
+
@config,
|
52
|
+
readers.compact,
|
53
|
+
writers.compact)
|
54
|
+
set_process_name(container.name)
|
55
|
+
container.run
|
59
56
|
rescue Errno::EPIPE # Broken Pipe, no problem
|
60
57
|
# Ignore
|
61
58
|
ensure
|
62
|
-
|
59
|
+
container.finalize if container
|
63
60
|
end
|
64
61
|
end
|
65
62
|
end
|
66
63
|
|
67
|
-
def spawn_command(a_shell_command, env: {},
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
writers << writer
|
81
|
-
end
|
82
|
-
|
83
|
-
|
84
|
-
shell_tokens = case a_shell_command
|
85
|
-
when Array
|
86
|
-
a_shell_command
|
87
|
-
else
|
88
|
-
a_shell_command.to_s.split(/\s+/)
|
89
|
-
end
|
90
|
-
|
91
|
-
inputs = readers.map { |it| create_pipe(it) }
|
92
|
-
outputs = writers.map { |it| create_pipe(it) }
|
93
|
-
|
94
|
-
tokens = shell_tokens.map do |token|
|
95
|
-
case token
|
96
|
-
when input_pattern
|
97
|
-
inputs.shift || fail('Missing reader')
|
98
|
-
when output_pattern
|
99
|
-
outputs.shift || fail('Missing writer')
|
100
|
-
else
|
101
|
-
token.to_s
|
64
|
+
def spawn_command(a_shell_command, env: {}, readers: [last_pipe], writers: [next_pipe])
|
65
|
+
writers.compact.each { |w| Channel.of_type(w).create(w, @config) }
|
66
|
+
@pids << fork do
|
67
|
+
begin
|
68
|
+
container = SoCommandContainer.new(a_shell_command,
|
69
|
+
env,
|
70
|
+
@config,
|
71
|
+
readers.compact,
|
72
|
+
writers.compact)
|
73
|
+
set_process_name(container.name)
|
74
|
+
container.run
|
75
|
+
rescue Errno::EPIPE # Broken Pipe, no problem
|
76
|
+
# Ignore
|
102
77
|
end
|
103
78
|
end
|
104
|
-
|
105
|
-
in_out = {close_others: true,
|
106
|
-
in: inputs.size == 1 ? [inputs.first, 'r'] : :close,
|
107
|
-
out: outputs.size == 1 ? [outputs.first, 'w'] : :close}
|
108
|
-
|
109
|
-
description = "#{tokens.first} #{in_out}"
|
110
|
-
@pids << fork { exec(env, [tokens.first, description], *tokens[1..-1], in_out) }
|
111
79
|
end
|
112
80
|
|
113
81
|
private
|
114
82
|
|
115
|
-
def append_open_io_if_needed(an_object)
|
116
|
-
class << an_object
|
117
|
-
prepend OpenIo
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
def create_pipe(pipe_name)
|
122
|
-
Channel.create_pipe(pipe_name)
|
123
|
-
end
|
124
|
-
|
125
83
|
def wait_children
|
126
|
-
# Don't wait if we have no children
|
127
|
-
# Thread.start do
|
128
|
-
# while @pids.size > 1
|
129
|
-
# @pids.each do |pid|
|
130
|
-
# begin
|
131
|
-
# Process.wait(pid, Process::WNOHANG)
|
132
|
-
# rescue Errno::ECHILD
|
133
|
-
# puts "Try delete #{pid}"
|
134
|
-
# @pids.delete(pid)
|
135
|
-
# end
|
136
|
-
# end
|
137
|
-
# sleep(0.5)
|
138
|
-
# end
|
139
|
-
# end
|
140
84
|
Process.waitall
|
141
|
-
@pids.clear
|
142
85
|
end
|
143
86
|
|
144
87
|
def kill_children
|
145
88
|
gpid = Process.gid
|
146
89
|
Process.kill('SIGTERM', -gpid) rescue nil
|
147
90
|
Timeout::timeout(5) { wait_children }
|
91
|
+
@pids.clear
|
148
92
|
rescue Timeout::Error
|
149
93
|
Process.kill('SIGKILL', -gpid) rescue nil
|
150
94
|
end
|
151
95
|
|
152
|
-
def set_process_name(name
|
153
|
-
$0 =
|
96
|
+
def set_process_name(name)
|
97
|
+
$0 = name
|
154
98
|
end
|
155
99
|
end
|
156
100
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class RubyObjectContainer
|
4
|
+
attr_reader :name
|
5
|
+
|
6
|
+
def initialize(class_or_object, args, process_name, config, reader_names, writers_names)
|
7
|
+
@args = args
|
8
|
+
@config = config
|
9
|
+
@reader_names = reader_names
|
10
|
+
@writer_names = writers_names
|
11
|
+
|
12
|
+
@the_object = prepare_object(class_or_object)
|
13
|
+
@name = prepare_process_name(@the_object, process_name, reader_names, writers_names)
|
14
|
+
end
|
15
|
+
|
16
|
+
def run
|
17
|
+
readers = @reader_names.map { |r| Channel.new(r, @config) }
|
18
|
+
writers = @writer_names.map { |w| Channel.new(w, @config) }
|
19
|
+
@the_object.run(*@args, readers: readers, writers: writers)
|
20
|
+
ensure
|
21
|
+
writers.each { |w| w.close if w } if writers
|
22
|
+
readers.each { |r| r.close if r } if readers
|
23
|
+
end
|
24
|
+
|
25
|
+
def finalize
|
26
|
+
@the_object.finalize if @the_object.respond_to?(:finalize)
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def prepare_object(class_or_object)
|
32
|
+
Class === class_or_object ? class_or_object.new : class_or_object
|
33
|
+
end
|
34
|
+
|
35
|
+
def prepare_process_name(the_object, process_name, reader_names, writer_names)
|
36
|
+
return process_name if process_name
|
37
|
+
name = the_object.respond_to?(:name) ? the_object.name : the_object.to_s
|
38
|
+
format('(%s)-->[%s]-->(%s)', reader_names.join(' '), name, writer_names.join(' '))
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class SoCommandContainer
|
4
|
+
INPUT_PATTERN = '((<))'
|
5
|
+
OUTPUT_PATTERN = '((>))'
|
6
|
+
attr_reader :name
|
7
|
+
|
8
|
+
def initialize(a_shell_command, env, config, reader_names, writer_names)
|
9
|
+
@env = env
|
10
|
+
@readers = reader_names.map { |r| Channel.of_type(r).io_name_for(r, config) }
|
11
|
+
@writers = writer_names.map { |w| Channel.of_type(w).io_name_for(w, config) }
|
12
|
+
@tokens = prepare_command(a_shell_command)
|
13
|
+
@name = @tokens.first +
|
14
|
+
@readers.map { |it| " <#{it}" }.join('') +
|
15
|
+
@writers.map { |it| " >#{it}" }.join('')
|
16
|
+
end
|
17
|
+
|
18
|
+
def run
|
19
|
+
in_out = {close_others: true,
|
20
|
+
in: @readers.size == 1 ? [@readers.first, 'r'] : :close,
|
21
|
+
out: @writers.size == 1 ? [@writers.first, 'w'] : :close}
|
22
|
+
|
23
|
+
exec(@env, [@tokens.first, name], *@tokens[1..-1], in_out)
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def prepare_command(a_shell_command)
|
29
|
+
shell_tokens = shell_tokens(a_shell_command)
|
30
|
+
replace_placeholders(shell_tokens)
|
31
|
+
end
|
32
|
+
|
33
|
+
def shell_tokens(a_shell_command)
|
34
|
+
case a_shell_command
|
35
|
+
when Array
|
36
|
+
a_shell_command
|
37
|
+
else
|
38
|
+
a_shell_command.to_s.split(/\s+/)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def replace_placeholders(shell_tokens)
|
43
|
+
shell_tokens.map do |token|
|
44
|
+
case token
|
45
|
+
when INPUT_PATTERN
|
46
|
+
@readers.shift || fail('Missing reader')
|
47
|
+
when OUTPUT_PATTERN
|
48
|
+
@writers.shift || fail('Missing writer')
|
49
|
+
else
|
50
|
+
token.to_s
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -6,8 +6,8 @@ module DeadlySerious
|
|
6
6
|
writer_param = params.any? { |(k, n)| k == :keyreq && n == :writer }
|
7
7
|
reader_param = params.any? { |(k, n)| k == :keyreq && n == :reader }
|
8
8
|
|
9
|
-
reader = JsonIo.new(readers.first) if readers.size == 1
|
10
|
-
writer = JsonIo.new(writers.first) if writers.size == 1
|
9
|
+
reader = JsonIo.new(readers.first) if readers.size == 1
|
10
|
+
writer = JsonIo.new(writers.first) if writers.size == 1
|
11
11
|
|
12
12
|
if reader_param && writer_param
|
13
13
|
unless reader
|
@@ -25,6 +25,8 @@ module DeadlySerious
|
|
25
25
|
# however, it's awesomely useful. =\
|
26
26
|
reader.each do |data|
|
27
27
|
result = block.call(*data)
|
28
|
+
|
29
|
+
# noinspection RubySimplifyBooleanInspection
|
28
30
|
if result == true # really TRUE, not thruthy
|
29
31
|
# Acts as filter
|
30
32
|
writer << data
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
include DeadlySerious::Engine
|
3
|
+
|
4
|
+
describe AutoPipe do
|
5
|
+
subject { AutoPipe.new }
|
6
|
+
describe '#last' do
|
7
|
+
it 'returns last writer' do
|
8
|
+
subject.next
|
9
|
+
expect(subject.last).to eq 'pipe.0001'
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'returns nil if no last writer' do
|
13
|
+
expect(subject.last).to be_nil
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#next' do
|
18
|
+
it 'returns next writer name' do
|
19
|
+
expect(subject.next).to eq 'pipe.0001'
|
20
|
+
expect(subject.next).to eq 'pipe.0002'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
describe '#on_subnet' do
|
25
|
+
it 'creates "subnames" to avoid conflicts' do
|
26
|
+
subject.next
|
27
|
+
subject.on_subnet do
|
28
|
+
expect(subject.next).to eq 'pipe.0001.0001'
|
29
|
+
expect(subject.last).to eq 'pipe.0001.0001'
|
30
|
+
expect(subject.next).to eq 'pipe.0001.0002'
|
31
|
+
subject.on_subnet do
|
32
|
+
expect(subject.next).to eq 'pipe.0001.0002.0001'
|
33
|
+
end
|
34
|
+
end
|
35
|
+
subject.next
|
36
|
+
subject.on_subnet do
|
37
|
+
expect(subject.next).to eq 'pipe.0002.0001'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
include DeadlySerious::Engine
|
3
|
+
|
4
|
+
describe SocketChannel do
|
5
|
+
let(:test_config) do
|
6
|
+
Config.new(data_dir: "/tmp/test_deadly_serious#{Process.pid}",
|
7
|
+
pipe_dir: "/tmp/test_deadly_serious#{Process.pid}",
|
8
|
+
preserve_pipe_dir: false)
|
9
|
+
end
|
10
|
+
|
11
|
+
matcher :produce_a do |expected|
|
12
|
+
subject = nil
|
13
|
+
match do |actual|
|
14
|
+
begin
|
15
|
+
subject = Channel.new(actual, test_config)
|
16
|
+
subject.is_a?(expected)
|
17
|
+
ensure
|
18
|
+
subject.close if subject
|
19
|
+
end
|
20
|
+
end
|
21
|
+
failure_message do
|
22
|
+
"but was a #{subject.class.name}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe 'sink' do
|
27
|
+
let(:snd) { '>}' }
|
28
|
+
let(:rcv) { '<}' }
|
29
|
+
|
30
|
+
def send_msg(*msgs, port: 5555)
|
31
|
+
fork do
|
32
|
+
sender = Channel.new("#{snd}localhost:#{port}", nil)
|
33
|
+
msgs.each { |m| sender << m }
|
34
|
+
sender.close
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'detects sender by its name' do
|
39
|
+
expect("#{snd}localhost:5555").to produce_a SocketSinkSendr
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'detects receiver by its name' do
|
43
|
+
expect("#{rcv}localhost:5555").to produce_a SocketSinkRecvr
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'connect two simple processes' do
|
47
|
+
begin
|
48
|
+
send_msg(1, 2)
|
49
|
+
receiver = Channel.new("#{rcv}localhost:5555", nil)
|
50
|
+
stream = receiver.each
|
51
|
+
expect(stream.next).to eq '1'
|
52
|
+
expect(stream.next).to eq '2'
|
53
|
+
ensure
|
54
|
+
receiver.close if receiver
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'ventilator' do
|
60
|
+
let(:snd) { '>{' }
|
61
|
+
let(:rcv) { '<{' }
|
62
|
+
|
63
|
+
def send_msg(*msgs, port: 5555)
|
64
|
+
fork do
|
65
|
+
sender = Channel.new("#{snd}localhost:#{port}", nil)
|
66
|
+
msgs.each { |m| sender << m }
|
67
|
+
sender.close
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'detects sender by its name' do
|
72
|
+
expect("#{snd}localhost:5555").to produce_a SocketVentSendr
|
73
|
+
end
|
74
|
+
|
75
|
+
it 'detects receiver by its name' do
|
76
|
+
expect("#{rcv}localhost:5555").to produce_a SocketVentRecvr
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'connect two simple processes' do
|
80
|
+
begin
|
81
|
+
send_msg(1, 2)
|
82
|
+
receiver = Channel.new("#{rcv}localhost:5555", nil)
|
83
|
+
stream = receiver.each
|
84
|
+
expect(stream.next).to eq '1'
|
85
|
+
expect(stream.next).to eq '2'
|
86
|
+
ensure
|
87
|
+
receiver.close if receiver
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
it 'has the same ZMQ context for multiple channels' do
|
92
|
+
c1 = Channel.new("#{rcv}localhost:5555", nil)
|
93
|
+
ctx1 = c1.context
|
94
|
+
|
95
|
+
c2 = Channel.new("#{snd}localhost:5556", nil)
|
96
|
+
ctx2 = c2.context
|
97
|
+
|
98
|
+
expect(ctx1).to eq ctx2
|
99
|
+
c1.close
|
100
|
+
c2.close
|
101
|
+
end
|
102
|
+
|
103
|
+
it 'has different contexts when open/close twice' do
|
104
|
+
c1 = Channel.new("#{rcv}localhost:5555", nil)
|
105
|
+
ctx1 = c1.context
|
106
|
+
c1.close
|
107
|
+
|
108
|
+
c2 = Channel.new("#{snd}localhost:5556", nil)
|
109
|
+
ctx2 = c2.context
|
110
|
+
c2.close
|
111
|
+
|
112
|
+
expect(ctx1).not_to eq ctx2
|
113
|
+
end
|
114
|
+
|
115
|
+
it 'clear ZMQ context on finish a single channel' do
|
116
|
+
send_msg(1)
|
117
|
+
channel = Channel.new("#{rcv}localhost:5555", nil)
|
118
|
+
channel.each {}
|
119
|
+
ctx = channel.context
|
120
|
+
channel.close
|
121
|
+
expect do
|
122
|
+
ctx.bind(:PUSH, 'tcp://*:5556')
|
123
|
+
end.to raise_error(ZMQ::Error, /has been destroyed/)
|
124
|
+
end
|
125
|
+
|
126
|
+
it 'clear ZMQ context on finish all channels' do
|
127
|
+
channel1 = Channel.new("#{rcv}localhost:5555", nil)
|
128
|
+
channel2 = Channel.new("#{rcv}localhost:5556", nil)
|
129
|
+
|
130
|
+
ctx = channel1.context
|
131
|
+
|
132
|
+
channel1.close
|
133
|
+
expect do
|
134
|
+
ctx.bind(:PUSH, 'tcp://*:5557')
|
135
|
+
end.not_to raise_error
|
136
|
+
|
137
|
+
channel2.close
|
138
|
+
expect do
|
139
|
+
ctx.bind(:PUSH, 'tcp://*:5558')
|
140
|
+
end.to raise_error(ZMQ::Error, /has been destroyed/)
|
141
|
+
end
|
142
|
+
|
143
|
+
it 'does load balancing' do
|
144
|
+
send_msg(1, 2, 3)
|
145
|
+
channel1 = Channel.new("#{rcv}localhost:5555", nil)
|
146
|
+
channel2 = Channel.new("#{rcv}localhost:5555", nil)
|
147
|
+
|
148
|
+
c1 = channel1.each
|
149
|
+
c2 = channel2.each
|
150
|
+
|
151
|
+
expect(c1.next).to eq '1'
|
152
|
+
expect(c2.next).to eq '2'
|
153
|
+
expect(c1.next).to eq '3'
|
154
|
+
|
155
|
+
channel1.close
|
156
|
+
channel2.close
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|