deadly_serious 1.0.2 → 2.0.0.pre.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/Guardfile +2 -2
  3. data/deadly_serious.gemspec +1 -0
  4. data/lib/deadly_serious.rb +7 -6
  5. data/lib/deadly_serious/engine/auto_pipe.rb +33 -21
  6. data/lib/deadly_serious/engine/channel.rb +15 -130
  7. data/lib/deadly_serious/engine/channel/file_channel.rb +50 -0
  8. data/lib/deadly_serious/engine/channel/pipe_channel.rb +59 -0
  9. data/lib/deadly_serious/engine/channel/socket/master_mind.rb +42 -0
  10. data/lib/deadly_serious/engine/channel/socket/minion.rb +24 -0
  11. data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb +34 -0
  12. data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb +26 -0
  13. data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb +32 -0
  14. data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb +30 -0
  15. data/lib/deadly_serious/engine/channel/socket_channel.rb +75 -0
  16. data/lib/deadly_serious/engine/commands.rb +29 -8
  17. data/lib/deadly_serious/engine/config.rb +55 -0
  18. data/lib/deadly_serious/engine/file_monitor.rb +57 -0
  19. data/lib/deadly_serious/engine/json_io.rb +13 -11
  20. data/lib/deadly_serious/engine/pipeline.rb +31 -87
  21. data/lib/deadly_serious/engine/ruby_object_container.rb +42 -0
  22. data/lib/deadly_serious/engine/so_command_container.rb +56 -0
  23. data/lib/deadly_serious/processes/converter.rb +12 -0
  24. data/lib/deadly_serious/processes/lambda.rb +4 -2
  25. data/lib/deadly_serious/processes/resilient_splitter.rb +1 -1
  26. data/lib/deadly_serious/version.rb +1 -1
  27. data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb +41 -0
  28. data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb +159 -0
  29. data/spec/{deadly_serious → lib/deadly_serious}/engine/commands_spec.rb +0 -0
  30. data/spec/lib/deadly_serious/engine/file_monitor_spec.rb +69 -0
  31. data/spec/{deadly_serious → lib/deadly_serious}/engine/json_io_spec.rb +0 -0
  32. data/spec/{deadly_serious → lib/deadly_serious}/engine/pipeline_spec.rb +37 -40
  33. data/spec/spec_helper.rb +4 -1
  34. metadata +51 -14
  35. data/lib/deadly_serious/engine/lazy_io.rb +0 -82
  36. data/lib/deadly_serious/engine/open_io.rb +0 -39
  37. data/lib/deadly_serious/processes/joiner.rb +0 -15
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c84a9aa66532d1b19c770ebb1d226855c073fd03
4
- data.tar.gz: 9390c5043f135abd7c6a7c02d713853770cc51f8
3
+ metadata.gz: 78ea6f0fb905d3b08feef450651add4e28b7ed69
4
+ data.tar.gz: 3bfb8bba7ca19b826ed1cfdefbb81cde2453eb33
5
5
  SHA512:
6
- metadata.gz: 2e70c78e33a1ccbdab49e4bca8dff082f63bcae6f3ee57d0586bd46c2b1cf1ad3212e9d881f25b543ee334dcaa34f74810c4f6fc03a1ac9a2406e850f6d670d8
7
- data.tar.gz: a7aede2aa595009185a0752906d9958925fd50cc367d5835fb566b49fc41ce1d84f0f2fd03accba0c6ec68eb57d0df85db83dcb96d2123b8f1bd906729805ca8
6
+ metadata.gz: 83cccbcf342d6e331f8bb050157446fef59a680e5968a8d00a1b21323d976d20f301a5844e899f1031d4cc1d0e049a04b47f7c11eef5214a58f1f552e8f05c6b
7
+ data.tar.gz: 3afb1d45bfc69a4721fb981949bb63e1b02f7a3765a417b8fe9eb2ad5adb9d60a4d81f51e3194d45a61466ebfd02e8040b5666e21768c4e10aca365b527901f2
data/Guardfile CHANGED
@@ -1,5 +1,5 @@
1
- guard :rspec, { all_on_start: true, keep: true, all_after_pass: true, run_all: { cmd: 'rspec -f progress' } } do
2
- watch(%r{^spec/.+_spec\.rb$})
1
+ guard :rspec, cmd: 'bundle exec rspec', all_on_start: true, keep: true, all_after_pass: true, run_all: { cmd: 'bundle exec rspec -f progress' } do
2
+ watch(%r{^spec/.+_spec\.rb$})
3
3
  watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" }
4
4
  watch('spec/spec_helper.rb') { "spec" }
5
5
  watch('lib/deadly_serious.rb') { "spec" }
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
25
25
 
26
26
  spec.add_dependency 'oj', '~> 2.0'
27
27
  spec.add_dependency 'multi_json', '~> 1.0'
28
+ spec.add_dependency 'rbczmq', '~> 1.7', '>= 1.7.9'
28
29
  end
@@ -1,17 +1,18 @@
1
1
  require 'multi_json'
2
2
  require 'fileutils'
3
- require 'shellwords'
3
+ require 'rb-inotify'
4
+ require 'rbczmq'
4
5
  require 'deadly_serious/version'
5
- require 'deadly_serious/engine/open_io'
6
+ require 'deadly_serious/engine/file_monitor'
6
7
  require 'deadly_serious/engine/json_io'
7
8
  require 'deadly_serious/engine/channel'
8
9
  require 'deadly_serious/engine/auto_pipe'
9
10
  require 'deadly_serious/engine/commands'
10
- require 'deadly_serious/engine/pipeline'
11
11
 
12
- Dir[File.dirname(__FILE__) + '/deadly_serious/engine/*.rb'].each do |file|
13
- require File.dirname(file) + '/' + File.basename(file, File.extname(file))
14
- end
12
+ require 'deadly_serious/engine/config'
13
+ require 'deadly_serious/engine/ruby_object_container'
14
+ require 'deadly_serious/engine/so_command_container'
15
+ require 'deadly_serious/engine/pipeline'
15
16
 
16
17
  # Loading all predefined processes
17
18
  Dir[File.join(File.dirname(__FILE__), 'deadly_serious', 'processes', '*.rb')].each do |file|
@@ -1,39 +1,51 @@
1
1
  class DeadlySerious::Engine::AutoPipe
2
- TEMPLATE = '%s.connection.%04d'
2
+ TEMPLATE = 'pipe.%s'
3
3
 
4
- def initialize
5
- @net_id = 0
6
- @connection_stack = []
7
- @counter = Hash.new { |h, k| h[k.to_sym] = 0}
8
- end
4
+ class Counter
5
+ TEMPLATE = '%04d'
9
6
 
10
- def on_subnet
11
- @net_id += 1
12
- @connection_stack << sprintf('%04d', @net_id)
13
- yield
14
- ensure
15
- @connection_stack.pop
16
- end
7
+ def initialize
8
+ @counter = 0
9
+ end
10
+
11
+ def next
12
+ @counter += 1
13
+ last
14
+ end
15
+
16
+ def last
17
+ format(TEMPLATE, @counter)
18
+ end
17
19
 
18
- def net_id
19
- (@connection_stack.last || 'top').to_sym
20
+ def zero?
21
+ @counter == 0
22
+ end
20
23
  end
21
24
 
22
- def counter
23
- @counter[net_id]
25
+ def initialize
26
+ @counters = [Counter.new]
24
27
  end
25
28
 
26
29
  def next
27
- advance_counter
30
+ current_counter.next
28
31
  last
29
32
  end
30
33
 
31
34
  def last
32
- sprintf(TEMPLATE, net_id, counter)
35
+ return nil if current_counter.zero?
36
+ format(TEMPLATE, @counters.map(&:last).join('.'))
37
+ end
38
+
39
+ def on_subnet
40
+ @counters << Counter.new
41
+ yield
42
+ ensure
43
+ @counters.pop
33
44
  end
34
45
 
35
46
  private
36
- def advance_counter
37
- @counter[net_id] += 1
47
+
48
+ def current_counter
49
+ @counters.last
38
50
  end
39
51
  end
@@ -1,140 +1,25 @@
1
- require 'socket'
2
- require 'deadly_serious/engine/lazy_io'
1
+ require 'deadly_serious/engine/channel/socket_channel'
2
+ require 'deadly_serious/engine/channel/file_channel'
3
+ require 'deadly_serious/engine/channel/pipe_channel'
3
4
 
4
5
  module DeadlySerious
5
6
  module Engine
6
- # Fake class, it's actually a factory ¬¬
7
- module Channel
8
- def self.new(name)
9
- matcher = name.match(/^(>)?(.*?)(?:(:)(\d{1,5}))?$/)
10
- if matcher[1] == '>'
11
- FileChannel.new(matcher[2], @data_dir)
12
- elsif matcher[3] == ':'
13
- SocketChannel.new(matcher[2], matcher[4].to_i)
14
- else
15
- PipeChannel.new(matcher[2], @pipe_dir)
16
- end
17
- end
18
-
19
- def self.config(data_dir, pipe_dir, preserve_pipe_dir)
20
- @data_dir = data_dir
21
- @pipe_dir = pipe_dir
22
- @preserve_pipe_dir = preserve_pipe_dir
23
- end
24
-
25
- def self.setup
26
- FileUtils.mkdir_p(@pipe_dir) unless File.exist?(@pipe_dir)
27
- end
28
-
29
- def self.teardown
30
- if !@preserve_pipe_dir && File.exist?(@pipe_dir)
31
- FileUtils.rm_r(@pipe_dir, force: true, secure: true)
32
- end
33
- end
34
-
35
- def self.create_pipe(pipe_name)
36
- new(pipe_name).create
37
- end
38
- end
39
7
 
40
- class FileChannel
41
- attr_reader :io_name
42
-
43
- def initialize(name, directory)
44
- if name =~ /^\//
45
- # Absolute file path
46
- @io_name = name
47
- else
48
- # relative file path (relative to data_dir)
49
- @io_name = File.join(directory, name)
50
- end
51
- end
52
-
53
- def create
54
- `touch '#{@io_name}'` unless File.exist?(@io_name)
55
- @io_name
56
- end
57
-
58
- def open_reader
59
- fail %(File "#{@io_name}" not found) unless File.exist?(@io_name)
60
- open(@io_name, 'r')
61
- end
62
-
63
- def open_writer
64
- fail %(File "#{@io_name}" not found) unless File.exist?(@io_name)
65
- open(@io_name, 'w')
66
- end
67
-
68
- def io
69
- LazyIo.new(self)
70
- end
71
- end
8
+ CHANNELS = [SocketChannel, FileChannel, PipeChannel]
72
9
 
73
- class PipeChannel
74
- attr_reader :io_name
75
-
76
- def initialize(name, directory)
77
- if name =~ /^\//
78
- # Absolute pipe path
79
- @io_name = name
80
- else
81
- # relative pipe path (relative to pipe_dir)
82
- @io_name = File.join(directory, name)
83
- end
84
- end
85
-
86
- def create
87
- `mkfifo '#{@io_name}'` unless File.exist?(@io_name)
88
- @io_name
89
- end
90
-
91
- def open_reader
92
- fail %(Pipe "#{@io_name}" not found) unless File.exist?(@io_name)
93
- open(@io_name, 'r')
94
- end
95
-
96
- def open_writer
97
- fail %(Pipe "#{@io_name}" not found) unless File.exist?(@io_name)
98
- open(@io_name, 'w')
99
- end
100
-
101
- def io
102
- LazyIo.new(self)
103
- end
104
- end
105
-
106
- class SocketChannel
107
- def initialize(host, port)
108
- @host, @port = host, port
109
- @retry_counter = 3
110
- end
111
-
112
- def io_name
113
- "#{@host}@#{@port}"
114
- end
115
-
116
- def create
117
- # Do nothing
118
- end
119
-
120
- def open_reader
121
- TCPSocket.new(@host, @port)
122
- rescue Exception => e
123
- @retry_counter -= 1
124
- if @retry_counter > 0
125
- sleep 1 and retry
126
- else
127
- raise e
128
- end
129
- end
130
-
131
- def open_writer
132
- server = TCPServer.new(@port)
133
- server.accept
10
+ # Fake class, it's actually a factory ¬¬
11
+ #
12
+ # name = '>xxx' # File
13
+ # name = 'xxx' # Pipe
14
+ # name = 'xxx:999' # Socket
15
+ # name = '!xxx:999' # 0MQueue
16
+ module Channel
17
+ def self.new(name, config)
18
+ of_type(name).new(name, config)
134
19
  end
135
20
 
136
- def io
137
- LazyIo.new(self)
21
+ def self.of_type(name)
22
+ CHANNELS.map { |channel| channel.of_type(name) }.compact.first
138
23
  end
139
24
  end
140
25
  end
@@ -0,0 +1,50 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class FileChannel
4
+ include Enumerable
5
+
6
+ REGEXP = /\A>(.*?)\z/
7
+
8
+ attr_reader :io_name
9
+
10
+ def self.of_type(name)
11
+ self if name.match(REGEXP)
12
+ end
13
+
14
+ def initialize(name, config)
15
+ @io_name = self.class.io_name_for(name, config)
16
+ end
17
+
18
+ def each
19
+ return enum_for(:each) unless block_given?
20
+ open(io_name, 'r') { |file| file.each_line { |line| yield line } }
21
+ end
22
+
23
+ def <<(data)
24
+ @writer ||= open(@io_name, 'w')
25
+ @writer.print(data)
26
+ self
27
+ end
28
+
29
+ def flush
30
+ @writer.flush if @writer
31
+ end
32
+
33
+ def close
34
+ @writer.close if @writer
35
+ end
36
+
37
+ def self.io_name_for(name, config)
38
+ matcher = name.match(REGEXP)
39
+ file_name = matcher[1]
40
+ config.file_path_for(file_name)
41
+ end
42
+
43
+ def self.create(name, config)
44
+ io_name = io_name_for(name, config)
45
+ `touch '#{io_name}'` unless File.exist?(io_name)
46
+ io_name
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,59 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class PipeChannel
4
+ include Enumerable
5
+
6
+ REGEXP = /\A(.*?)\z/
7
+
8
+ attr_reader :io_name
9
+
10
+ def self.of_type(name)
11
+ self if name.match(REGEXP)
12
+ end
13
+
14
+ def initialize(name, config)
15
+ @io_name = self.class.io_name_for(name, config)
16
+ end
17
+
18
+ def each
19
+ return enum_for(:each) unless block_given?
20
+ open(io_name, 'r') { |file| file.each_line { |line| yield line } }
21
+ end
22
+
23
+ def <<(data)
24
+ @writer ||= open(@io_name, File::WRONLY)
25
+ @writer.print(data)
26
+ self
27
+ end
28
+
29
+ def flush
30
+ @writer.flush if @writer
31
+ end
32
+
33
+ def close
34
+ @writer.close if @writer
35
+ end
36
+
37
+ def self.io_name_for(name, config)
38
+ matcher = name.match(REGEXP)
39
+ pipe_name = matcher[1]
40
+ config.pipe_path_for(pipe_name)
41
+ end
42
+
43
+ def self.create(name, config)
44
+ # Redirecting to /dev/null when we test the
45
+ # file existence. It STILL tries to create
46
+ # the pipe due concurrency :(
47
+ # Not good >_<
48
+ io_name = io_name_for(name, config)
49
+ `mkfifo '#{io_name}' 2>/dev/null` unless File.exist?(io_name)
50
+ io_name
51
+ end
52
+
53
+ private
54
+
55
+ def retries(times = 3, sleep = 0.5)
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,42 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class MasterMind
4
+ attr_reader :factory
5
+
6
+ def self.new_instance
7
+ pid = Process.pid
8
+ if @master.nil? || @master[:pid] != pid
9
+ @master = {pid: pid, master: self.new}
10
+ end
11
+ @master[:master]
12
+ end
13
+
14
+ def initialize
15
+ @factory = ZMQ::Context.new
16
+ @counter = 1
17
+ @minions = []
18
+ end
19
+
20
+ def spawn_minion
21
+ minion_brain = yield(@factory, @counter)
22
+ @counter += 1
23
+ Minion.new(self, minion_brain).tap { |m| @minions << m }
24
+ rescue ZMQ::Error => e
25
+ raise if e.message !~ /has been destroyed/
26
+ @factory = ZMQ::Context.new
27
+ retry
28
+ end
29
+
30
+ def destroy_body_of(minion)
31
+ @minions.delete(minion)
32
+ suicide if @minions.empty?
33
+ end
34
+
35
+ private
36
+
37
+ def suicide
38
+ @factory.destroy
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,24 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class Minion
4
+ extend Forwardable
5
+
6
+ def_delegators :@brain, :send, :recv
7
+
8
+ def initialize(mastermind, brain)
9
+ @mastermind = mastermind
10
+ @brain = brain
11
+ end
12
+
13
+ def send_to(destiny, msg)
14
+ @brain.sendm(destiny)
15
+ @brain.send(msg)
16
+ end
17
+
18
+ def explode
19
+ @brain.close
20
+ @mastermind.destroy_body_of(self)
21
+ end
22
+ end
23
+ end
24
+ end