deadly_serious 1.0.2 → 2.0.0.pre.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Guardfile +2 -2
- data/deadly_serious.gemspec +1 -0
- data/lib/deadly_serious.rb +7 -6
- data/lib/deadly_serious/engine/auto_pipe.rb +33 -21
- data/lib/deadly_serious/engine/channel.rb +15 -130
- data/lib/deadly_serious/engine/channel/file_channel.rb +50 -0
- data/lib/deadly_serious/engine/channel/pipe_channel.rb +59 -0
- data/lib/deadly_serious/engine/channel/socket/master_mind.rb +42 -0
- data/lib/deadly_serious/engine/channel/socket/minion.rb +24 -0
- data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb +34 -0
- data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb +26 -0
- data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb +32 -0
- data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb +30 -0
- data/lib/deadly_serious/engine/channel/socket_channel.rb +75 -0
- data/lib/deadly_serious/engine/commands.rb +29 -8
- data/lib/deadly_serious/engine/config.rb +55 -0
- data/lib/deadly_serious/engine/file_monitor.rb +57 -0
- data/lib/deadly_serious/engine/json_io.rb +13 -11
- data/lib/deadly_serious/engine/pipeline.rb +31 -87
- data/lib/deadly_serious/engine/ruby_object_container.rb +42 -0
- data/lib/deadly_serious/engine/so_command_container.rb +56 -0
- data/lib/deadly_serious/processes/converter.rb +12 -0
- data/lib/deadly_serious/processes/lambda.rb +4 -2
- data/lib/deadly_serious/processes/resilient_splitter.rb +1 -1
- data/lib/deadly_serious/version.rb +1 -1
- data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb +41 -0
- data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb +159 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/commands_spec.rb +0 -0
- data/spec/lib/deadly_serious/engine/file_monitor_spec.rb +69 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/json_io_spec.rb +0 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/pipeline_spec.rb +37 -40
- data/spec/spec_helper.rb +4 -1
- metadata +51 -14
- data/lib/deadly_serious/engine/lazy_io.rb +0 -82
- data/lib/deadly_serious/engine/open_io.rb +0 -39
- data/lib/deadly_serious/processes/joiner.rb +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 78ea6f0fb905d3b08feef450651add4e28b7ed69
|
4
|
+
data.tar.gz: 3bfb8bba7ca19b826ed1cfdefbb81cde2453eb33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 83cccbcf342d6e331f8bb050157446fef59a680e5968a8d00a1b21323d976d20f301a5844e899f1031d4cc1d0e049a04b47f7c11eef5214a58f1f552e8f05c6b
|
7
|
+
data.tar.gz: 3afb1d45bfc69a4721fb981949bb63e1b02f7a3765a417b8fe9eb2ad5adb9d60a4d81f51e3194d45a61466ebfd02e8040b5666e21768c4e10aca365b527901f2
|
data/Guardfile
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
guard :rspec,
|
2
|
-
|
1
|
+
guard :rspec, cmd: 'bundle exec rspec', all_on_start: true, keep: true, all_after_pass: true, run_all: { cmd: 'bundle exec rspec -f progress' } do
|
2
|
+
watch(%r{^spec/.+_spec\.rb$})
|
3
3
|
watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" }
|
4
4
|
watch('spec/spec_helper.rb') { "spec" }
|
5
5
|
watch('lib/deadly_serious.rb') { "spec" }
|
data/deadly_serious.gemspec
CHANGED
data/lib/deadly_serious.rb
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
require 'multi_json'
|
2
2
|
require 'fileutils'
|
3
|
-
require '
|
3
|
+
require 'rb-inotify'
|
4
|
+
require 'rbczmq'
|
4
5
|
require 'deadly_serious/version'
|
5
|
-
require 'deadly_serious/engine/
|
6
|
+
require 'deadly_serious/engine/file_monitor'
|
6
7
|
require 'deadly_serious/engine/json_io'
|
7
8
|
require 'deadly_serious/engine/channel'
|
8
9
|
require 'deadly_serious/engine/auto_pipe'
|
9
10
|
require 'deadly_serious/engine/commands'
|
10
|
-
require 'deadly_serious/engine/pipeline'
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
require 'deadly_serious/engine/config'
|
13
|
+
require 'deadly_serious/engine/ruby_object_container'
|
14
|
+
require 'deadly_serious/engine/so_command_container'
|
15
|
+
require 'deadly_serious/engine/pipeline'
|
15
16
|
|
16
17
|
# Loading all predefined processes
|
17
18
|
Dir[File.join(File.dirname(__FILE__), 'deadly_serious', 'processes', '*.rb')].each do |file|
|
@@ -1,39 +1,51 @@
|
|
1
1
|
class DeadlySerious::Engine::AutoPipe
|
2
|
-
TEMPLATE = '
|
2
|
+
TEMPLATE = 'pipe.%s'
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
@connection_stack = []
|
7
|
-
@counter = Hash.new { |h, k| h[k.to_sym] = 0}
|
8
|
-
end
|
4
|
+
class Counter
|
5
|
+
TEMPLATE = '%04d'
|
9
6
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
7
|
+
def initialize
|
8
|
+
@counter = 0
|
9
|
+
end
|
10
|
+
|
11
|
+
def next
|
12
|
+
@counter += 1
|
13
|
+
last
|
14
|
+
end
|
15
|
+
|
16
|
+
def last
|
17
|
+
format(TEMPLATE, @counter)
|
18
|
+
end
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
+
def zero?
|
21
|
+
@counter == 0
|
22
|
+
end
|
20
23
|
end
|
21
24
|
|
22
|
-
def
|
23
|
-
@
|
25
|
+
def initialize
|
26
|
+
@counters = [Counter.new]
|
24
27
|
end
|
25
28
|
|
26
29
|
def next
|
27
|
-
|
30
|
+
current_counter.next
|
28
31
|
last
|
29
32
|
end
|
30
33
|
|
31
34
|
def last
|
32
|
-
|
35
|
+
return nil if current_counter.zero?
|
36
|
+
format(TEMPLATE, @counters.map(&:last).join('.'))
|
37
|
+
end
|
38
|
+
|
39
|
+
def on_subnet
|
40
|
+
@counters << Counter.new
|
41
|
+
yield
|
42
|
+
ensure
|
43
|
+
@counters.pop
|
33
44
|
end
|
34
45
|
|
35
46
|
private
|
36
|
-
|
37
|
-
|
47
|
+
|
48
|
+
def current_counter
|
49
|
+
@counters.last
|
38
50
|
end
|
39
51
|
end
|
@@ -1,140 +1,25 @@
|
|
1
|
-
require '
|
2
|
-
require 'deadly_serious/engine/
|
1
|
+
require 'deadly_serious/engine/channel/socket_channel'
|
2
|
+
require 'deadly_serious/engine/channel/file_channel'
|
3
|
+
require 'deadly_serious/engine/channel/pipe_channel'
|
3
4
|
|
4
5
|
module DeadlySerious
|
5
6
|
module Engine
|
6
|
-
# Fake class, it's actually a factory ¬¬
|
7
|
-
module Channel
|
8
|
-
def self.new(name)
|
9
|
-
matcher = name.match(/^(>)?(.*?)(?:(:)(\d{1,5}))?$/)
|
10
|
-
if matcher[1] == '>'
|
11
|
-
FileChannel.new(matcher[2], @data_dir)
|
12
|
-
elsif matcher[3] == ':'
|
13
|
-
SocketChannel.new(matcher[2], matcher[4].to_i)
|
14
|
-
else
|
15
|
-
PipeChannel.new(matcher[2], @pipe_dir)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
def self.config(data_dir, pipe_dir, preserve_pipe_dir)
|
20
|
-
@data_dir = data_dir
|
21
|
-
@pipe_dir = pipe_dir
|
22
|
-
@preserve_pipe_dir = preserve_pipe_dir
|
23
|
-
end
|
24
|
-
|
25
|
-
def self.setup
|
26
|
-
FileUtils.mkdir_p(@pipe_dir) unless File.exist?(@pipe_dir)
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.teardown
|
30
|
-
if !@preserve_pipe_dir && File.exist?(@pipe_dir)
|
31
|
-
FileUtils.rm_r(@pipe_dir, force: true, secure: true)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
def self.create_pipe(pipe_name)
|
36
|
-
new(pipe_name).create
|
37
|
-
end
|
38
|
-
end
|
39
7
|
|
40
|
-
|
41
|
-
attr_reader :io_name
|
42
|
-
|
43
|
-
def initialize(name, directory)
|
44
|
-
if name =~ /^\//
|
45
|
-
# Absolute file path
|
46
|
-
@io_name = name
|
47
|
-
else
|
48
|
-
# relative file path (relative to data_dir)
|
49
|
-
@io_name = File.join(directory, name)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def create
|
54
|
-
`touch '#{@io_name}'` unless File.exist?(@io_name)
|
55
|
-
@io_name
|
56
|
-
end
|
57
|
-
|
58
|
-
def open_reader
|
59
|
-
fail %(File "#{@io_name}" not found) unless File.exist?(@io_name)
|
60
|
-
open(@io_name, 'r')
|
61
|
-
end
|
62
|
-
|
63
|
-
def open_writer
|
64
|
-
fail %(File "#{@io_name}" not found) unless File.exist?(@io_name)
|
65
|
-
open(@io_name, 'w')
|
66
|
-
end
|
67
|
-
|
68
|
-
def io
|
69
|
-
LazyIo.new(self)
|
70
|
-
end
|
71
|
-
end
|
8
|
+
CHANNELS = [SocketChannel, FileChannel, PipeChannel]
|
72
9
|
|
73
|
-
class
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
@io_name = File.join(directory, name)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
def create
|
87
|
-
`mkfifo '#{@io_name}'` unless File.exist?(@io_name)
|
88
|
-
@io_name
|
89
|
-
end
|
90
|
-
|
91
|
-
def open_reader
|
92
|
-
fail %(Pipe "#{@io_name}" not found) unless File.exist?(@io_name)
|
93
|
-
open(@io_name, 'r')
|
94
|
-
end
|
95
|
-
|
96
|
-
def open_writer
|
97
|
-
fail %(Pipe "#{@io_name}" not found) unless File.exist?(@io_name)
|
98
|
-
open(@io_name, 'w')
|
99
|
-
end
|
100
|
-
|
101
|
-
def io
|
102
|
-
LazyIo.new(self)
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
class SocketChannel
|
107
|
-
def initialize(host, port)
|
108
|
-
@host, @port = host, port
|
109
|
-
@retry_counter = 3
|
110
|
-
end
|
111
|
-
|
112
|
-
def io_name
|
113
|
-
"#{@host}@#{@port}"
|
114
|
-
end
|
115
|
-
|
116
|
-
def create
|
117
|
-
# Do nothing
|
118
|
-
end
|
119
|
-
|
120
|
-
def open_reader
|
121
|
-
TCPSocket.new(@host, @port)
|
122
|
-
rescue Exception => e
|
123
|
-
@retry_counter -= 1
|
124
|
-
if @retry_counter > 0
|
125
|
-
sleep 1 and retry
|
126
|
-
else
|
127
|
-
raise e
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
def open_writer
|
132
|
-
server = TCPServer.new(@port)
|
133
|
-
server.accept
|
10
|
+
# Fake class, it's actually a factory ¬¬
|
11
|
+
#
|
12
|
+
# name = '>xxx' # File
|
13
|
+
# name = 'xxx' # Pipe
|
14
|
+
# name = 'xxx:999' # Socket
|
15
|
+
# name = '!xxx:999' # 0MQueue
|
16
|
+
module Channel
|
17
|
+
def self.new(name, config)
|
18
|
+
of_type(name).new(name, config)
|
134
19
|
end
|
135
20
|
|
136
|
-
def
|
137
|
-
|
21
|
+
def self.of_type(name)
|
22
|
+
CHANNELS.map { |channel| channel.of_type(name) }.compact.first
|
138
23
|
end
|
139
24
|
end
|
140
25
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class FileChannel
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
REGEXP = /\A>(.*?)\z/
|
7
|
+
|
8
|
+
attr_reader :io_name
|
9
|
+
|
10
|
+
def self.of_type(name)
|
11
|
+
self if name.match(REGEXP)
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(name, config)
|
15
|
+
@io_name = self.class.io_name_for(name, config)
|
16
|
+
end
|
17
|
+
|
18
|
+
def each
|
19
|
+
return enum_for(:each) unless block_given?
|
20
|
+
open(io_name, 'r') { |file| file.each_line { |line| yield line } }
|
21
|
+
end
|
22
|
+
|
23
|
+
def <<(data)
|
24
|
+
@writer ||= open(@io_name, 'w')
|
25
|
+
@writer.print(data)
|
26
|
+
self
|
27
|
+
end
|
28
|
+
|
29
|
+
def flush
|
30
|
+
@writer.flush if @writer
|
31
|
+
end
|
32
|
+
|
33
|
+
def close
|
34
|
+
@writer.close if @writer
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.io_name_for(name, config)
|
38
|
+
matcher = name.match(REGEXP)
|
39
|
+
file_name = matcher[1]
|
40
|
+
config.file_path_for(file_name)
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.create(name, config)
|
44
|
+
io_name = io_name_for(name, config)
|
45
|
+
`touch '#{io_name}'` unless File.exist?(io_name)
|
46
|
+
io_name
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class PipeChannel
|
4
|
+
include Enumerable
|
5
|
+
|
6
|
+
REGEXP = /\A(.*?)\z/
|
7
|
+
|
8
|
+
attr_reader :io_name
|
9
|
+
|
10
|
+
def self.of_type(name)
|
11
|
+
self if name.match(REGEXP)
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(name, config)
|
15
|
+
@io_name = self.class.io_name_for(name, config)
|
16
|
+
end
|
17
|
+
|
18
|
+
def each
|
19
|
+
return enum_for(:each) unless block_given?
|
20
|
+
open(io_name, 'r') { |file| file.each_line { |line| yield line } }
|
21
|
+
end
|
22
|
+
|
23
|
+
def <<(data)
|
24
|
+
@writer ||= open(@io_name, File::WRONLY)
|
25
|
+
@writer.print(data)
|
26
|
+
self
|
27
|
+
end
|
28
|
+
|
29
|
+
def flush
|
30
|
+
@writer.flush if @writer
|
31
|
+
end
|
32
|
+
|
33
|
+
def close
|
34
|
+
@writer.close if @writer
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.io_name_for(name, config)
|
38
|
+
matcher = name.match(REGEXP)
|
39
|
+
pipe_name = matcher[1]
|
40
|
+
config.pipe_path_for(pipe_name)
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.create(name, config)
|
44
|
+
# Redirecting to /dev/null when we test the
|
45
|
+
# file existence. It STILL tries to create
|
46
|
+
# the pipe due concurrency :(
|
47
|
+
# Not good >_<
|
48
|
+
io_name = io_name_for(name, config)
|
49
|
+
`mkfifo '#{io_name}' 2>/dev/null` unless File.exist?(io_name)
|
50
|
+
io_name
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def retries(times = 3, sleep = 0.5)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class MasterMind
|
4
|
+
attr_reader :factory
|
5
|
+
|
6
|
+
def self.new_instance
|
7
|
+
pid = Process.pid
|
8
|
+
if @master.nil? || @master[:pid] != pid
|
9
|
+
@master = {pid: pid, master: self.new}
|
10
|
+
end
|
11
|
+
@master[:master]
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
@factory = ZMQ::Context.new
|
16
|
+
@counter = 1
|
17
|
+
@minions = []
|
18
|
+
end
|
19
|
+
|
20
|
+
def spawn_minion
|
21
|
+
minion_brain = yield(@factory, @counter)
|
22
|
+
@counter += 1
|
23
|
+
Minion.new(self, minion_brain).tap { |m| @minions << m }
|
24
|
+
rescue ZMQ::Error => e
|
25
|
+
raise if e.message !~ /has been destroyed/
|
26
|
+
@factory = ZMQ::Context.new
|
27
|
+
retry
|
28
|
+
end
|
29
|
+
|
30
|
+
def destroy_body_of(minion)
|
31
|
+
@minions.delete(minion)
|
32
|
+
suicide if @minions.empty?
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
def suicide
|
38
|
+
@factory.destroy
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class Minion
|
4
|
+
extend Forwardable
|
5
|
+
|
6
|
+
def_delegators :@brain, :send, :recv
|
7
|
+
|
8
|
+
def initialize(mastermind, brain)
|
9
|
+
@mastermind = mastermind
|
10
|
+
@brain = brain
|
11
|
+
end
|
12
|
+
|
13
|
+
def send_to(destiny, msg)
|
14
|
+
@brain.sendm(destiny)
|
15
|
+
@brain.send(msg)
|
16
|
+
end
|
17
|
+
|
18
|
+
def explode
|
19
|
+
@brain.close
|
20
|
+
@mastermind.destroy_body_of(self)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|