deadly_serious 1.0.2 → 2.0.0.pre.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/Guardfile +2 -2
  3. data/deadly_serious.gemspec +1 -0
  4. data/lib/deadly_serious.rb +7 -6
  5. data/lib/deadly_serious/engine/auto_pipe.rb +33 -21
  6. data/lib/deadly_serious/engine/channel.rb +15 -130
  7. data/lib/deadly_serious/engine/channel/file_channel.rb +50 -0
  8. data/lib/deadly_serious/engine/channel/pipe_channel.rb +59 -0
  9. data/lib/deadly_serious/engine/channel/socket/master_mind.rb +42 -0
  10. data/lib/deadly_serious/engine/channel/socket/minion.rb +24 -0
  11. data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb +34 -0
  12. data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb +26 -0
  13. data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb +32 -0
  14. data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb +30 -0
  15. data/lib/deadly_serious/engine/channel/socket_channel.rb +75 -0
  16. data/lib/deadly_serious/engine/commands.rb +29 -8
  17. data/lib/deadly_serious/engine/config.rb +55 -0
  18. data/lib/deadly_serious/engine/file_monitor.rb +57 -0
  19. data/lib/deadly_serious/engine/json_io.rb +13 -11
  20. data/lib/deadly_serious/engine/pipeline.rb +31 -87
  21. data/lib/deadly_serious/engine/ruby_object_container.rb +42 -0
  22. data/lib/deadly_serious/engine/so_command_container.rb +56 -0
  23. data/lib/deadly_serious/processes/converter.rb +12 -0
  24. data/lib/deadly_serious/processes/lambda.rb +4 -2
  25. data/lib/deadly_serious/processes/resilient_splitter.rb +1 -1
  26. data/lib/deadly_serious/version.rb +1 -1
  27. data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb +41 -0
  28. data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb +159 -0
  29. data/spec/{deadly_serious → lib/deadly_serious}/engine/commands_spec.rb +0 -0
  30. data/spec/lib/deadly_serious/engine/file_monitor_spec.rb +69 -0
  31. data/spec/{deadly_serious → lib/deadly_serious}/engine/json_io_spec.rb +0 -0
  32. data/spec/{deadly_serious → lib/deadly_serious}/engine/pipeline_spec.rb +37 -40
  33. data/spec/spec_helper.rb +4 -1
  34. metadata +51 -14
  35. data/lib/deadly_serious/engine/lazy_io.rb +0 -82
  36. data/lib/deadly_serious/engine/open_io.rb +0 -39
  37. data/lib/deadly_serious/processes/joiner.rb +0 -15
@@ -3,28 +3,27 @@ module DeadlySerious
3
3
  class Pipeline
4
4
  include DeadlySerious::Engine::Commands
5
5
 
6
- attr_reader :data_dir, :pipe_dir, :pids
6
+ attr_reader :pids, :config
7
7
 
8
8
  def initialize(data_dir: './data',
9
9
  pipe_dir: "/tmp/deadly_serious/#{Process.pid}",
10
10
  preserve_pipe_dir: false,
11
11
  &block)
12
- @data_dir = data_dir
13
- @pipe_dir = pipe_dir
12
+
13
+ @config = Config.new(data_dir: data_dir, pipe_dir: pipe_dir, preserve_pipe_dir: preserve_pipe_dir)
14
14
  @block = block
15
15
  @pids = []
16
- Channel.config(data_dir, pipe_dir, preserve_pipe_dir)
17
16
  end
18
17
 
19
18
  def run
20
- Channel.setup
19
+ @config.setup
21
20
  @block.call(self)
22
21
  wait_children
23
22
  rescue => e
24
23
  kill_children
25
24
  raise e
26
25
  ensure
27
- Channel.teardown
26
+ @config.teardown if @config
28
27
  end
29
28
 
30
29
  # Wait all sub processes to finish before
@@ -43,114 +42,59 @@ module DeadlySerious
43
42
  # prefer the simpler {DeadlySerious::Engine::Commands#spawn_class} or
44
43
  # the {DeadlySerious::Engine::Commands#spawn} methods.
45
44
  def spawn_process(class_or_object, *args, process_name: nil, readers: [last_pipe], writers: [next_pipe])
46
- # TODO if we have no readers, alarm! (how about data sources???)
47
- # TODO if we have no readers, and this is the first process, read from STDIN
48
- # TODO if we have no writers, alarm! (how about data sinks???)
49
- # TODO if we have no writers, and this is the last process, write to STDOUT
50
- process_name ||= class_or_object.respond_to?(:name) ? class_or_object.name : class_or_object.to_s
51
- writers.each { |writer| create_pipe(writer) }
45
+ writers.compact.each { |w| Channel.of_type(w).create(w, @config) }
52
46
  @pids << fork do
53
47
  begin
54
- set_process_name(process_name, readers, writers)
55
- # TODO Change this to not modify "a_class", so we can pass instances too
56
- the_object = Class === class_or_object ? class_or_object.new : class_or_object
57
- append_open_io_if_needed(the_object)
58
- the_object.run(*args, readers: readers, writers: writers)
48
+ container = RubyObjectContainer.new(class_or_object,
49
+ args,
50
+ process_name,
51
+ @config,
52
+ readers.compact,
53
+ writers.compact)
54
+ set_process_name(container.name)
55
+ container.run
59
56
  rescue Errno::EPIPE # Broken Pipe, no problem
60
57
  # Ignore
61
58
  ensure
62
- the_object.finalize if the_object.respond_to?(:finalize)
59
+ container.finalize if container
63
60
  end
64
61
  end
65
62
  end
66
63
 
67
- def spawn_command(a_shell_command, env: {}, reader: nil, writer: nil, readers: [], writers: [])
68
- input_pattern = '((<))'
69
- output_pattern = '((>))'
70
-
71
- if reader.nil? && readers.empty?
72
- readers << last_pipe
73
- elsif reader && readers.empty?
74
- readers << reader
75
- end
76
-
77
- if writer.nil? && writers.empty?
78
- writers << next_pipe
79
- elsif writer && writers.empty?
80
- writers << writer
81
- end
82
-
83
-
84
- shell_tokens = case a_shell_command
85
- when Array
86
- a_shell_command
87
- else
88
- a_shell_command.to_s.split(/\s+/)
89
- end
90
-
91
- inputs = readers.map { |it| create_pipe(it) }
92
- outputs = writers.map { |it| create_pipe(it) }
93
-
94
- tokens = shell_tokens.map do |token|
95
- case token
96
- when input_pattern
97
- inputs.shift || fail('Missing reader')
98
- when output_pattern
99
- outputs.shift || fail('Missing writer')
100
- else
101
- token.to_s
64
+ def spawn_command(a_shell_command, env: {}, readers: [last_pipe], writers: [next_pipe])
65
+ writers.compact.each { |w| Channel.of_type(w).create(w, @config) }
66
+ @pids << fork do
67
+ begin
68
+ container = SoCommandContainer.new(a_shell_command,
69
+ env,
70
+ @config,
71
+ readers.compact,
72
+ writers.compact)
73
+ set_process_name(container.name)
74
+ container.run
75
+ rescue Errno::EPIPE # Broken Pipe, no problem
76
+ # Ignore
102
77
  end
103
78
  end
104
-
105
- in_out = {close_others: true,
106
- in: inputs.size == 1 ? [inputs.first, 'r'] : :close,
107
- out: outputs.size == 1 ? [outputs.first, 'w'] : :close}
108
-
109
- description = "#{tokens.first} #{in_out}"
110
- @pids << fork { exec(env, [tokens.first, description], *tokens[1..-1], in_out) }
111
79
  end
112
80
 
113
81
  private
114
82
 
115
- def append_open_io_if_needed(an_object)
116
- class << an_object
117
- prepend OpenIo
118
- end
119
- end
120
-
121
- def create_pipe(pipe_name)
122
- Channel.create_pipe(pipe_name)
123
- end
124
-
125
83
  def wait_children
126
- # Don't wait if we have no children
127
- # Thread.start do
128
- # while @pids.size > 1
129
- # @pids.each do |pid|
130
- # begin
131
- # Process.wait(pid, Process::WNOHANG)
132
- # rescue Errno::ECHILD
133
- # puts "Try delete #{pid}"
134
- # @pids.delete(pid)
135
- # end
136
- # end
137
- # sleep(0.5)
138
- # end
139
- # end
140
84
  Process.waitall
141
- @pids.clear
142
85
  end
143
86
 
144
87
  def kill_children
145
88
  gpid = Process.gid
146
89
  Process.kill('SIGTERM', -gpid) rescue nil
147
90
  Timeout::timeout(5) { wait_children }
91
+ @pids.clear
148
92
  rescue Timeout::Error
149
93
  Process.kill('SIGKILL', -gpid) rescue nil
150
94
  end
151
95
 
152
- def set_process_name(name, readers, writers)
153
- $0 = format('(%s)-->[%s]-->(%s)', readers.join(', '), name, writers.join(' '))
96
+ def set_process_name(name)
97
+ $0 = name
154
98
  end
155
99
  end
156
100
  end
@@ -0,0 +1,42 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class RubyObjectContainer
4
+ attr_reader :name
5
+
6
+ def initialize(class_or_object, args, process_name, config, reader_names, writers_names)
7
+ @args = args
8
+ @config = config
9
+ @reader_names = reader_names
10
+ @writer_names = writers_names
11
+
12
+ @the_object = prepare_object(class_or_object)
13
+ @name = prepare_process_name(@the_object, process_name, reader_names, writers_names)
14
+ end
15
+
16
+ def run
17
+ readers = @reader_names.map { |r| Channel.new(r, @config) }
18
+ writers = @writer_names.map { |w| Channel.new(w, @config) }
19
+ @the_object.run(*@args, readers: readers, writers: writers)
20
+ ensure
21
+ writers.each { |w| w.close if w } if writers
22
+ readers.each { |r| r.close if r } if readers
23
+ end
24
+
25
+ def finalize
26
+ @the_object.finalize if @the_object.respond_to?(:finalize)
27
+ end
28
+
29
+ private
30
+
31
+ def prepare_object(class_or_object)
32
+ Class === class_or_object ? class_or_object.new : class_or_object
33
+ end
34
+
35
+ def prepare_process_name(the_object, process_name, reader_names, writer_names)
36
+ return process_name if process_name
37
+ name = the_object.respond_to?(:name) ? the_object.name : the_object.to_s
38
+ format('(%s)-->[%s]-->(%s)', reader_names.join(' '), name, writer_names.join(' '))
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,56 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class SoCommandContainer
4
+ INPUT_PATTERN = '((<))'
5
+ OUTPUT_PATTERN = '((>))'
6
+ attr_reader :name
7
+
8
+ def initialize(a_shell_command, env, config, reader_names, writer_names)
9
+ @env = env
10
+ @readers = reader_names.map { |r| Channel.of_type(r).io_name_for(r, config) }
11
+ @writers = writer_names.map { |w| Channel.of_type(w).io_name_for(w, config) }
12
+ @tokens = prepare_command(a_shell_command)
13
+ @name = @tokens.first +
14
+ @readers.map { |it| " <#{it}" }.join('') +
15
+ @writers.map { |it| " >#{it}" }.join('')
16
+ end
17
+
18
+ def run
19
+ in_out = {close_others: true,
20
+ in: @readers.size == 1 ? [@readers.first, 'r'] : :close,
21
+ out: @writers.size == 1 ? [@writers.first, 'w'] : :close}
22
+
23
+ exec(@env, [@tokens.first, name], *@tokens[1..-1], in_out)
24
+ end
25
+
26
+ private
27
+
28
+ def prepare_command(a_shell_command)
29
+ shell_tokens = shell_tokens(a_shell_command)
30
+ replace_placeholders(shell_tokens)
31
+ end
32
+
33
+ def shell_tokens(a_shell_command)
34
+ case a_shell_command
35
+ when Array
36
+ a_shell_command
37
+ else
38
+ a_shell_command.to_s.split(/\s+/)
39
+ end
40
+ end
41
+
42
+ def replace_placeholders(shell_tokens)
43
+ shell_tokens.map do |token|
44
+ case token
45
+ when INPUT_PATTERN
46
+ @readers.shift || fail('Missing reader')
47
+ when OUTPUT_PATTERN
48
+ @writers.shift || fail('Missing writer')
49
+ else
50
+ token.to_s
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,12 @@
1
+ module DeadlySerious
2
+ module Processes
3
+ class Converter
4
+ def run(readers: [], writers: [])
5
+ reader = readers.first
6
+ reader.each do |line|
7
+ writers.each { |w| w << line }
8
+ end
9
+ end
10
+ end
11
+ end
12
+ end
@@ -6,8 +6,8 @@ module DeadlySerious
6
6
  writer_param = params.any? { |(k, n)| k == :keyreq && n == :writer }
7
7
  reader_param = params.any? { |(k, n)| k == :keyreq && n == :reader }
8
8
 
9
- reader = JsonIo.new(readers.first) if readers.size == 1 && File.exist?(readers.first.filename)
10
- writer = JsonIo.new(writers.first) if writers.size == 1 && File.exist?(writers.first.filename)
9
+ reader = JsonIo.new(readers.first) if readers.size == 1
10
+ writer = JsonIo.new(writers.first) if writers.size == 1
11
11
 
12
12
  if reader_param && writer_param
13
13
  unless reader
@@ -25,6 +25,8 @@ module DeadlySerious
25
25
  # however, it's awesomely useful. =\
26
26
  reader.each do |data|
27
27
  result = block.call(*data)
28
+
29
+ # noinspection RubySimplifyBooleanInspection
28
30
  if result == true # really TRUE, not thruthy
29
31
  # Acts as filter
30
32
  writer << data
@@ -19,7 +19,7 @@ module DeadlySerious
19
19
  outputs = writers.dup
20
20
  end
21
21
  current = outputs.first
22
- current << line
22
+ current << line << "\n"
23
23
  outputs.rotate!
24
24
  rescue Errno::EPIPE => e
25
25
  puts e.inspect
@@ -1,3 +1,3 @@
1
1
  module DeadlySerious
2
- VERSION = '1.0.2'
2
+ VERSION = '2.0.0-rc1'
3
3
  end
@@ -0,0 +1,41 @@
1
+ require 'spec_helper'
2
+ include DeadlySerious::Engine
3
+
4
+ describe AutoPipe do
5
+ subject { AutoPipe.new }
6
+ describe '#last' do
7
+ it 'returns last writer' do
8
+ subject.next
9
+ expect(subject.last).to eq 'pipe.0001'
10
+ end
11
+
12
+ it 'returns nil if no last writer' do
13
+ expect(subject.last).to be_nil
14
+ end
15
+ end
16
+
17
+ describe '#next' do
18
+ it 'returns next writer name' do
19
+ expect(subject.next).to eq 'pipe.0001'
20
+ expect(subject.next).to eq 'pipe.0002'
21
+ end
22
+ end
23
+
24
+ describe '#on_subnet' do
25
+ it 'creates "subnames" to avoid conflicts' do
26
+ subject.next
27
+ subject.on_subnet do
28
+ expect(subject.next).to eq 'pipe.0001.0001'
29
+ expect(subject.last).to eq 'pipe.0001.0001'
30
+ expect(subject.next).to eq 'pipe.0001.0002'
31
+ subject.on_subnet do
32
+ expect(subject.next).to eq 'pipe.0001.0002.0001'
33
+ end
34
+ end
35
+ subject.next
36
+ subject.on_subnet do
37
+ expect(subject.next).to eq 'pipe.0002.0001'
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,159 @@
1
+ require 'spec_helper'
2
+ include DeadlySerious::Engine
3
+
4
+ describe SocketChannel do
5
+ let(:test_config) do
6
+ Config.new(data_dir: "/tmp/test_deadly_serious#{Process.pid}",
7
+ pipe_dir: "/tmp/test_deadly_serious#{Process.pid}",
8
+ preserve_pipe_dir: false)
9
+ end
10
+
11
+ matcher :produce_a do |expected|
12
+ subject = nil
13
+ match do |actual|
14
+ begin
15
+ subject = Channel.new(actual, test_config)
16
+ subject.is_a?(expected)
17
+ ensure
18
+ subject.close if subject
19
+ end
20
+ end
21
+ failure_message do
22
+ "but was a #{subject.class.name}"
23
+ end
24
+ end
25
+
26
+ describe 'sink' do
27
+ let(:snd) { '>}' }
28
+ let(:rcv) { '<}' }
29
+
30
+ def send_msg(*msgs, port: 5555)
31
+ fork do
32
+ sender = Channel.new("#{snd}localhost:#{port}", nil)
33
+ msgs.each { |m| sender << m }
34
+ sender.close
35
+ end
36
+ end
37
+
38
+ it 'detects sender by its name' do
39
+ expect("#{snd}localhost:5555").to produce_a SocketSinkSendr
40
+ end
41
+
42
+ it 'detects receiver by its name' do
43
+ expect("#{rcv}localhost:5555").to produce_a SocketSinkRecvr
44
+ end
45
+
46
+ it 'connect two simple processes' do
47
+ begin
48
+ send_msg(1, 2)
49
+ receiver = Channel.new("#{rcv}localhost:5555", nil)
50
+ stream = receiver.each
51
+ expect(stream.next).to eq '1'
52
+ expect(stream.next).to eq '2'
53
+ ensure
54
+ receiver.close if receiver
55
+ end
56
+ end
57
+ end
58
+
59
+ describe 'ventilator' do
60
+ let(:snd) { '>{' }
61
+ let(:rcv) { '<{' }
62
+
63
+ def send_msg(*msgs, port: 5555)
64
+ fork do
65
+ sender = Channel.new("#{snd}localhost:#{port}", nil)
66
+ msgs.each { |m| sender << m }
67
+ sender.close
68
+ end
69
+ end
70
+
71
+ it 'detects sender by its name' do
72
+ expect("#{snd}localhost:5555").to produce_a SocketVentSendr
73
+ end
74
+
75
+ it 'detects receiver by its name' do
76
+ expect("#{rcv}localhost:5555").to produce_a SocketVentRecvr
77
+ end
78
+
79
+ it 'connect two simple processes' do
80
+ begin
81
+ send_msg(1, 2)
82
+ receiver = Channel.new("#{rcv}localhost:5555", nil)
83
+ stream = receiver.each
84
+ expect(stream.next).to eq '1'
85
+ expect(stream.next).to eq '2'
86
+ ensure
87
+ receiver.close if receiver
88
+ end
89
+ end
90
+
91
+ it 'has the same ZMQ context for multiple channels' do
92
+ c1 = Channel.new("#{rcv}localhost:5555", nil)
93
+ ctx1 = c1.context
94
+
95
+ c2 = Channel.new("#{snd}localhost:5556", nil)
96
+ ctx2 = c2.context
97
+
98
+ expect(ctx1).to eq ctx2
99
+ c1.close
100
+ c2.close
101
+ end
102
+
103
+ it 'has different contexts when open/close twice' do
104
+ c1 = Channel.new("#{rcv}localhost:5555", nil)
105
+ ctx1 = c1.context
106
+ c1.close
107
+
108
+ c2 = Channel.new("#{snd}localhost:5556", nil)
109
+ ctx2 = c2.context
110
+ c2.close
111
+
112
+ expect(ctx1).not_to eq ctx2
113
+ end
114
+
115
+ it 'clear ZMQ context on finish a single channel' do
116
+ send_msg(1)
117
+ channel = Channel.new("#{rcv}localhost:5555", nil)
118
+ channel.each {}
119
+ ctx = channel.context
120
+ channel.close
121
+ expect do
122
+ ctx.bind(:PUSH, 'tcp://*:5556')
123
+ end.to raise_error(ZMQ::Error, /has been destroyed/)
124
+ end
125
+
126
+ it 'clear ZMQ context on finish all channels' do
127
+ channel1 = Channel.new("#{rcv}localhost:5555", nil)
128
+ channel2 = Channel.new("#{rcv}localhost:5556", nil)
129
+
130
+ ctx = channel1.context
131
+
132
+ channel1.close
133
+ expect do
134
+ ctx.bind(:PUSH, 'tcp://*:5557')
135
+ end.not_to raise_error
136
+
137
+ channel2.close
138
+ expect do
139
+ ctx.bind(:PUSH, 'tcp://*:5558')
140
+ end.to raise_error(ZMQ::Error, /has been destroyed/)
141
+ end
142
+
143
+ it 'does load balancing' do
144
+ send_msg(1, 2, 3)
145
+ channel1 = Channel.new("#{rcv}localhost:5555", nil)
146
+ channel2 = Channel.new("#{rcv}localhost:5555", nil)
147
+
148
+ c1 = channel1.each
149
+ c2 = channel2.each
150
+
151
+ expect(c1.next).to eq '1'
152
+ expect(c2.next).to eq '2'
153
+ expect(c1.next).to eq '3'
154
+
155
+ channel1.close
156
+ channel2.close
157
+ end
158
+ end
159
+ end