deadly_serious 1.0.2 → 2.0.0.pre.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Guardfile +2 -2
- data/deadly_serious.gemspec +1 -0
- data/lib/deadly_serious.rb +7 -6
- data/lib/deadly_serious/engine/auto_pipe.rb +33 -21
- data/lib/deadly_serious/engine/channel.rb +15 -130
- data/lib/deadly_serious/engine/channel/file_channel.rb +50 -0
- data/lib/deadly_serious/engine/channel/pipe_channel.rb +59 -0
- data/lib/deadly_serious/engine/channel/socket/master_mind.rb +42 -0
- data/lib/deadly_serious/engine/channel/socket/minion.rb +24 -0
- data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb +34 -0
- data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb +26 -0
- data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb +32 -0
- data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb +30 -0
- data/lib/deadly_serious/engine/channel/socket_channel.rb +75 -0
- data/lib/deadly_serious/engine/commands.rb +29 -8
- data/lib/deadly_serious/engine/config.rb +55 -0
- data/lib/deadly_serious/engine/file_monitor.rb +57 -0
- data/lib/deadly_serious/engine/json_io.rb +13 -11
- data/lib/deadly_serious/engine/pipeline.rb +31 -87
- data/lib/deadly_serious/engine/ruby_object_container.rb +42 -0
- data/lib/deadly_serious/engine/so_command_container.rb +56 -0
- data/lib/deadly_serious/processes/converter.rb +12 -0
- data/lib/deadly_serious/processes/lambda.rb +4 -2
- data/lib/deadly_serious/processes/resilient_splitter.rb +1 -1
- data/lib/deadly_serious/version.rb +1 -1
- data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb +41 -0
- data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb +159 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/commands_spec.rb +0 -0
- data/spec/lib/deadly_serious/engine/file_monitor_spec.rb +69 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/json_io_spec.rb +0 -0
- data/spec/{deadly_serious → lib/deadly_serious}/engine/pipeline_spec.rb +37 -40
- data/spec/spec_helper.rb +4 -1
- metadata +51 -14
- data/lib/deadly_serious/engine/lazy_io.rb +0 -82
- data/lib/deadly_serious/engine/open_io.rb +0 -39
- data/lib/deadly_serious/processes/joiner.rb +0 -15
@@ -0,0 +1,34 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class SocketSinkRecvr < SocketChannel
|
4
|
+
|
5
|
+
attr_reader :io_name
|
6
|
+
|
7
|
+
def initialize(name, _config)
|
8
|
+
super
|
9
|
+
@io_name = format('tcp://*:%d', port)
|
10
|
+
@minion = master.spawn_minion { |ctx| ctx.bind(:PULL, @io_name) }
|
11
|
+
end
|
12
|
+
|
13
|
+
def each
|
14
|
+
return enum_for(:each) unless block_given?
|
15
|
+
clients = 0
|
16
|
+
loop do
|
17
|
+
msg = @minion.recv
|
18
|
+
if msg == END_MSG
|
19
|
+
clients -= 1
|
20
|
+
break if clients <= 0
|
21
|
+
elsif msg == RDY_MSG
|
22
|
+
clients += 1
|
23
|
+
else
|
24
|
+
yield msg
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def close
|
30
|
+
@minion.explode
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class SocketSinkSendr < SocketChannel
|
4
|
+
|
5
|
+
attr_reader :io_name
|
6
|
+
|
7
|
+
def initialize(name, _config)
|
8
|
+
super
|
9
|
+
@io_name = format('tcp://%s:%d', host, port)
|
10
|
+
@minion = master.spawn_minion { |ctx| ctx.connect(:PUSH, @io_name) }
|
11
|
+
sleep(0.5) # Avoid slow joiner syndrome the stupid way >(
|
12
|
+
@minion.send(RDY_MSG)
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(data)
|
16
|
+
@minion.send(data.to_s)
|
17
|
+
self
|
18
|
+
end
|
19
|
+
|
20
|
+
def close
|
21
|
+
@minion.send(END_MSG)
|
22
|
+
@minion.explode
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class SocketVentRecvr < SocketChannel
|
4
|
+
|
5
|
+
attr_reader :io_name
|
6
|
+
|
7
|
+
def initialize(name, _config)
|
8
|
+
super
|
9
|
+
@io_name = format('tcp://%s:%d', host, port)
|
10
|
+
@minion = master.spawn_minion do |ctx, counter|
|
11
|
+
socket = ctx.socket(:DEALER)
|
12
|
+
socket.identity = format('%d:%d', Process.pid, counter)
|
13
|
+
socket.connect(@io_name)
|
14
|
+
socket
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def each
|
19
|
+
return enum_for(:each) unless block_given?
|
20
|
+
@minion.send('') # I'm ready!
|
21
|
+
while (msg = @minion.recv) != END_MSG
|
22
|
+
yield msg
|
23
|
+
@minion.send('') # More msg, pls!
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def close
|
28
|
+
@minion.explode
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class SocketVentSendr < SocketChannel
|
4
|
+
|
5
|
+
attr_reader :io_name
|
6
|
+
|
7
|
+
def initialize(name, _config)
|
8
|
+
super
|
9
|
+
@io_name = format('tcp://*:%d', port)
|
10
|
+
@minion = master.spawn_minion { |ctx| ctx.bind(:ROUTER, @io_name) }
|
11
|
+
@receivers = Set.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def <<(data)
|
15
|
+
identity = @minion.recv # Stop until ready
|
16
|
+
@receivers << identity
|
17
|
+
@minion.recv # Discard message ("command")
|
18
|
+
@minion.send_to(identity, data.to_s)
|
19
|
+
self
|
20
|
+
end
|
21
|
+
|
22
|
+
def close
|
23
|
+
@receivers.each do |identity|
|
24
|
+
@minion.send_to(identity, END_MSG)
|
25
|
+
end
|
26
|
+
@minion.explode
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'deadly_serious/engine/channel/socket/minion'
|
2
|
+
require 'deadly_serious/engine/channel/socket/master_mind'
|
3
|
+
|
4
|
+
module DeadlySerious
|
5
|
+
module Engine
|
6
|
+
class SocketChannel
|
7
|
+
# Odd, but I had too :(
|
8
|
+
require 'deadly_serious/engine/channel/socket/socket_vent_recvr'
|
9
|
+
require 'deadly_serious/engine/channel/socket/socket_vent_sendr'
|
10
|
+
require 'deadly_serious/engine/channel/socket/socket_sink_recvr'
|
11
|
+
require 'deadly_serious/engine/channel/socket/socket_sink_sendr'
|
12
|
+
include Enumerable
|
13
|
+
|
14
|
+
END_MSG = 'END TRANSMISSION'.freeze
|
15
|
+
RDY_MSG = 'READY FOR TRANSMISSION'.freeze
|
16
|
+
|
17
|
+
DEFAULT_PORT = 10001
|
18
|
+
REGEXP = /\A([<>][{}])([^:]+):(\d{1,5})\z/
|
19
|
+
|
20
|
+
attr_reader :host, :port, :master
|
21
|
+
|
22
|
+
def self.of_type(name)
|
23
|
+
matcher = name.match(REGEXP)
|
24
|
+
return if matcher.nil?
|
25
|
+
type = matcher[1]
|
26
|
+
case type
|
27
|
+
when '>{'
|
28
|
+
SocketVentSendr
|
29
|
+
when '<{'
|
30
|
+
SocketVentRecvr
|
31
|
+
when '>}'
|
32
|
+
SocketSinkSendr
|
33
|
+
when '<}'
|
34
|
+
SocketSinkRecvr
|
35
|
+
else
|
36
|
+
nil
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def initialize(name, _config)
|
41
|
+
matcher = name.match(REGEXP)
|
42
|
+
host = matcher[2]
|
43
|
+
port = matcher[3]
|
44
|
+
@host = host.to_s.empty? ? 'localhost' : host.to_s
|
45
|
+
@port = port.to_s.empty? ? DEFAULT_PORT : port.to_i
|
46
|
+
@master = MasterMind.new_instance
|
47
|
+
end
|
48
|
+
|
49
|
+
def each
|
50
|
+
fail 'Subclass implementation'
|
51
|
+
end
|
52
|
+
|
53
|
+
def <<(_data)
|
54
|
+
fail 'Subclass implementation'
|
55
|
+
end
|
56
|
+
|
57
|
+
def close
|
58
|
+
fail 'Subclass implementation'
|
59
|
+
end
|
60
|
+
|
61
|
+
def flush
|
62
|
+
# Do nothing
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.create(_name, _config)
|
66
|
+
# Do nothing
|
67
|
+
end
|
68
|
+
|
69
|
+
# Only for tests
|
70
|
+
def context
|
71
|
+
master.factory
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -2,6 +2,7 @@ module DeadlySerious
|
|
2
2
|
module Engine
|
3
3
|
# Commands make work with Pipelines easier.
|
4
4
|
module Commands
|
5
|
+
MQUEUE_START_PORT = 13500
|
5
6
|
|
6
7
|
private def auto_pipe
|
7
8
|
@auto_pipe ||= AutoPipe.new
|
@@ -23,14 +24,14 @@ module DeadlySerious
|
|
23
24
|
# the next component.
|
24
25
|
def from_file(file_name, writer: next_pipe)
|
25
26
|
file = file_name.sub(/^>?(.*)$/, '>\1')
|
26
|
-
spawn_command('cat',
|
27
|
+
spawn_command('cat ((<))', readers: [file], writers: [writer])
|
27
28
|
end
|
28
29
|
|
29
30
|
# Write a file to "data" dir from the pipe
|
30
31
|
# of the last component
|
31
32
|
def to_file(file_name, reader: last_pipe)
|
32
33
|
file = file_name.sub(/^>?(.*)$/, '>\1')
|
33
|
-
spawn_command('cat',
|
34
|
+
spawn_command('cat', readers: [reader], writers: [file])
|
34
35
|
end
|
35
36
|
|
36
37
|
# Read from a specific named pipe.
|
@@ -38,7 +39,7 @@ module DeadlySerious
|
|
38
39
|
# This is useful after a {#spawn_tee}, sometimes.
|
39
40
|
def from_pipe(pipe_name, writer: next_pipe)
|
40
41
|
pipe = pipe_name.sub(/^>?/, '')
|
41
|
-
spawn_command('cat',
|
42
|
+
spawn_command('cat', readers: [pipe], writers: [writer])
|
42
43
|
end
|
43
44
|
|
44
45
|
# Write the output of the last component to
|
@@ -49,7 +50,7 @@ module DeadlySerious
|
|
49
50
|
# {#spawn_tee} instead.
|
50
51
|
def to_pipe(pipe_name, reader: last_pipe)
|
51
52
|
pipe = pipe_name.sub(/^>?/, '')
|
52
|
-
spawn_command('cat',
|
53
|
+
spawn_command('cat', readers: [reader], writers: [pipe])
|
53
54
|
end
|
54
55
|
|
55
56
|
# Spawn an object connected to the last and next components
|
@@ -65,6 +66,7 @@ module DeadlySerious
|
|
65
66
|
# Spawn {number_of_processes} classes, one process for each of them.
|
66
67
|
# Also, it divides the previous pipe in {number_of_processes} pipes,
|
67
68
|
# an routes data through them.
|
69
|
+
# @deprecated
|
68
70
|
def spawn_class_parallel(number_of_processes, class_name, *args, reader: last_pipe, writer: next_pipe)
|
69
71
|
connect_a = (1..number_of_processes).map { |i| sprintf('%s.%da.splitter', class_name.to_s.downcase.gsub(/\W+/, '_'), i) }
|
70
72
|
connect_b = (1..number_of_processes).map { |i| sprintf('%s.%db.splitter', class_name.to_s.downcase.gsub(/\W+/, '_'), i) }
|
@@ -75,8 +77,8 @@ module DeadlySerious
|
|
75
77
|
spawn_process(DeadlySerious::Processes::Joiner, readers: connect_b, writers: [writer])
|
76
78
|
end
|
77
79
|
|
78
|
-
def spawn_lambda(reader: last_pipe, writer: next_pipe, &block)
|
79
|
-
spawn_process(DeadlySerious::Processes::Lambda, block, readers: [reader], writers: [writer])
|
80
|
+
def spawn_lambda(name: 'Lambda',reader: last_pipe, writer: next_pipe, &block)
|
81
|
+
spawn_process(DeadlySerious::Processes::Lambda, block, process_name: name, readers: [reader], writers: [writer])
|
80
82
|
end
|
81
83
|
|
82
84
|
# Pipe from the last component to a intermediate
|
@@ -92,11 +94,13 @@ module DeadlySerious
|
|
92
94
|
|
93
95
|
if block_given?
|
94
96
|
on_subnet do
|
95
|
-
|
97
|
+
name = next_pipe
|
98
|
+
path = Channel.of_type(name).create(name, config)
|
99
|
+
spawn_command("tee #{path}", readers: [reader], writers: [writer])
|
96
100
|
block.call
|
97
101
|
end
|
98
102
|
elsif escape
|
99
|
-
spawn_command("tee #{
|
103
|
+
spawn_command("tee #{Channel.of_type(escape).io_name_for(escape, config)}", readers: [reader], writers: [writer])
|
100
104
|
else
|
101
105
|
fail 'No block or escape given'
|
102
106
|
end
|
@@ -114,6 +118,23 @@ module DeadlySerious
|
|
114
118
|
w.create
|
115
119
|
spawn("cat '#{r.io_name}' > '#{charger.io_name}' && cat '#{charger.io_name}' > '#{w.io_name}' && rm '#{charger.io_name}'")
|
116
120
|
end
|
121
|
+
|
122
|
+
# Distribute data to "number_of_lanes" sub pipelines
|
123
|
+
def parallel(number_of_lanes, reader: last_pipe, writer: next_pipe)
|
124
|
+
@port ||= MQUEUE_START_PORT
|
125
|
+
ventilator = format('>{localhost:%d', @port)
|
126
|
+
input = format('<{localhost:%d', @port)
|
127
|
+
@port += 1
|
128
|
+
sink = format('<}localhost:%d', @port)
|
129
|
+
output = format('>}localhost:%d', @port)
|
130
|
+
@port += 1
|
131
|
+
|
132
|
+
spawn(Processes::Converter.new, reader: reader, writer: ventilator)
|
133
|
+
spawn(Processes::Converter.new, reader: sink, writer: writer)
|
134
|
+
on_subnet do
|
135
|
+
number_of_lanes.times { yield input, output }
|
136
|
+
end
|
137
|
+
end
|
117
138
|
end
|
118
139
|
end
|
119
140
|
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class Config
|
4
|
+
attr_reader :data_dir, :pipe_dir, :preserve_pipe_dir
|
5
|
+
|
6
|
+
def initialize(data_dir:, pipe_dir:, preserve_pipe_dir:)
|
7
|
+
@data_dir = data_dir
|
8
|
+
@pipe_dir = pipe_dir
|
9
|
+
@preserve_pipe_dir = preserve_pipe_dir
|
10
|
+
end
|
11
|
+
|
12
|
+
def file_path_for(name)
|
13
|
+
path_for(data_dir, name)
|
14
|
+
end
|
15
|
+
|
16
|
+
def pipe_path_for(name)
|
17
|
+
path_for(pipe_dir, name)
|
18
|
+
end
|
19
|
+
|
20
|
+
def path_for(directory, name)
|
21
|
+
if name =~ /^\//
|
22
|
+
# Absolute file path
|
23
|
+
name
|
24
|
+
else
|
25
|
+
# relative file path (relative to data_dir)
|
26
|
+
File.join(directory, name)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def setup
|
31
|
+
create_data_dir
|
32
|
+
create_pipe_dir
|
33
|
+
end
|
34
|
+
|
35
|
+
def teardown
|
36
|
+
destroy_pipe_dir
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def create_data_dir
|
42
|
+
FileUtils.mkdir_p(@data_dir) unless File.exist?(@data_dir)
|
43
|
+
end
|
44
|
+
|
45
|
+
def create_pipe_dir
|
46
|
+
FileUtils.mkdir_p(@pipe_dir) unless File.exist?(@pipe_dir)
|
47
|
+
end
|
48
|
+
|
49
|
+
def destroy_pipe_dir
|
50
|
+
return if @preserve_pipe_dir || !File.exist?(@pipe_dir)
|
51
|
+
FileUtils.rm_r(@pipe_dir, force: true, secure: true)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module DeadlySerious
|
2
|
+
module Engine
|
3
|
+
class FileMonitor
|
4
|
+
class Parts
|
5
|
+
attr_reader :directory, :name
|
6
|
+
|
7
|
+
def initialize(file_name)
|
8
|
+
matcher = file_name.to_s.match(%r{\A((?<dir>.*)/)?(?<name>[^/]+)\z})
|
9
|
+
@directory = matcher[:dir]
|
10
|
+
@name = matcher[:name]
|
11
|
+
end
|
12
|
+
|
13
|
+
def exist?
|
14
|
+
File.exist?(to_s)
|
15
|
+
end
|
16
|
+
|
17
|
+
def to_s
|
18
|
+
File.join(@directory, @name)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(*file_names)
|
23
|
+
@parts = file_names.map { |f| Parts.new(f) }
|
24
|
+
end
|
25
|
+
|
26
|
+
def wait_creation
|
27
|
+
part = @parts.find { |p| p.exist? }
|
28
|
+
return part.to_s if part
|
29
|
+
watch_event(@parts, :create)
|
30
|
+
end
|
31
|
+
|
32
|
+
def wait_modification
|
33
|
+
notifier = INotify::Notifier.new
|
34
|
+
@parts.each { |p| notifier.watch(p.to_s, :modify) { Fiber.yield p.to_s } }
|
35
|
+
fiber = Fiber.new { notifier.process }
|
36
|
+
fiber.resume
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def watch_event(parts, event)
|
42
|
+
dirs = parts.group_by(&:directory)
|
43
|
+
notifier = INotify::Notifier.new
|
44
|
+
dirs.each do |dir, ps|
|
45
|
+
files = ps.map(&:name)
|
46
|
+
notifier.watch(dir, event) do |e|
|
47
|
+
Fiber.yield(File.join(dir, e.name)) if files.include?(e.name)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
fiber = Fiber.new { notifier.run }
|
51
|
+
file_name = fiber.resume
|
52
|
+
notifier.stop
|
53
|
+
file_name
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -8,29 +8,31 @@ module DeadlySerious
|
|
8
8
|
end
|
9
9
|
|
10
10
|
def each
|
11
|
-
|
12
|
-
|
13
|
-
else
|
14
|
-
@io.lazy.map { |line| parse_line(line) }
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def parse_line(line)
|
19
|
-
MultiJson.load(line)
|
11
|
+
return enum_for(:each) unless block_given?
|
12
|
+
@io.each { |line| yield parse(line) }
|
20
13
|
end
|
21
14
|
|
22
15
|
def <<(value)
|
23
16
|
case value
|
24
17
|
when Hash
|
25
|
-
@io << MultiJson.dump(value)
|
18
|
+
@io << "#{MultiJson.dump(value)}\n"
|
26
19
|
else
|
27
|
-
@io << MultiJson.dump(Array(value))
|
20
|
+
@io << "#{MultiJson.dump(Array(value))}\n"
|
28
21
|
end
|
29
22
|
end
|
30
23
|
|
31
24
|
def flush
|
32
25
|
@io.flush
|
33
26
|
end
|
27
|
+
|
28
|
+
private
|
29
|
+
def parse(line)
|
30
|
+
MultiJson.load(line)
|
31
|
+
rescue MultiJson::ParseError
|
32
|
+
puts 'Error in parse'
|
33
|
+
puts line
|
34
|
+
raise
|
35
|
+
end
|
34
36
|
end
|
35
37
|
end
|
36
38
|
end
|