deadly_serious 1.0.2 → 2.0.0.pre.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/Guardfile +2 -2
  3. data/deadly_serious.gemspec +1 -0
  4. data/lib/deadly_serious.rb +7 -6
  5. data/lib/deadly_serious/engine/auto_pipe.rb +33 -21
  6. data/lib/deadly_serious/engine/channel.rb +15 -130
  7. data/lib/deadly_serious/engine/channel/file_channel.rb +50 -0
  8. data/lib/deadly_serious/engine/channel/pipe_channel.rb +59 -0
  9. data/lib/deadly_serious/engine/channel/socket/master_mind.rb +42 -0
  10. data/lib/deadly_serious/engine/channel/socket/minion.rb +24 -0
  11. data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb +34 -0
  12. data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb +26 -0
  13. data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb +32 -0
  14. data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb +30 -0
  15. data/lib/deadly_serious/engine/channel/socket_channel.rb +75 -0
  16. data/lib/deadly_serious/engine/commands.rb +29 -8
  17. data/lib/deadly_serious/engine/config.rb +55 -0
  18. data/lib/deadly_serious/engine/file_monitor.rb +57 -0
  19. data/lib/deadly_serious/engine/json_io.rb +13 -11
  20. data/lib/deadly_serious/engine/pipeline.rb +31 -87
  21. data/lib/deadly_serious/engine/ruby_object_container.rb +42 -0
  22. data/lib/deadly_serious/engine/so_command_container.rb +56 -0
  23. data/lib/deadly_serious/processes/converter.rb +12 -0
  24. data/lib/deadly_serious/processes/lambda.rb +4 -2
  25. data/lib/deadly_serious/processes/resilient_splitter.rb +1 -1
  26. data/lib/deadly_serious/version.rb +1 -1
  27. data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb +41 -0
  28. data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb +159 -0
  29. data/spec/{deadly_serious → lib/deadly_serious}/engine/commands_spec.rb +0 -0
  30. data/spec/lib/deadly_serious/engine/file_monitor_spec.rb +69 -0
  31. data/spec/{deadly_serious → lib/deadly_serious}/engine/json_io_spec.rb +0 -0
  32. data/spec/{deadly_serious → lib/deadly_serious}/engine/pipeline_spec.rb +37 -40
  33. data/spec/spec_helper.rb +4 -1
  34. metadata +51 -14
  35. data/lib/deadly_serious/engine/lazy_io.rb +0 -82
  36. data/lib/deadly_serious/engine/open_io.rb +0 -39
  37. data/lib/deadly_serious/processes/joiner.rb +0 -15
@@ -0,0 +1,34 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class SocketSinkRecvr < SocketChannel
4
+
5
+ attr_reader :io_name
6
+
7
+ def initialize(name, _config)
8
+ super
9
+ @io_name = format('tcp://*:%d', port)
10
+ @minion = master.spawn_minion { |ctx| ctx.bind(:PULL, @io_name) }
11
+ end
12
+
13
+ def each
14
+ return enum_for(:each) unless block_given?
15
+ clients = 0
16
+ loop do
17
+ msg = @minion.recv
18
+ if msg == END_MSG
19
+ clients -= 1
20
+ break if clients <= 0
21
+ elsif msg == RDY_MSG
22
+ clients += 1
23
+ else
24
+ yield msg
25
+ end
26
+ end
27
+ end
28
+
29
+ def close
30
+ @minion.explode
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,26 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class SocketSinkSendr < SocketChannel
4
+
5
+ attr_reader :io_name
6
+
7
+ def initialize(name, _config)
8
+ super
9
+ @io_name = format('tcp://%s:%d', host, port)
10
+ @minion = master.spawn_minion { |ctx| ctx.connect(:PUSH, @io_name) }
11
+ sleep(0.5) # Avoid slow joiner syndrome the stupid way >(
12
+ @minion.send(RDY_MSG)
13
+ end
14
+
15
+ def <<(data)
16
+ @minion.send(data.to_s)
17
+ self
18
+ end
19
+
20
+ def close
21
+ @minion.send(END_MSG)
22
+ @minion.explode
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,32 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class SocketVentRecvr < SocketChannel
4
+
5
+ attr_reader :io_name
6
+
7
+ def initialize(name, _config)
8
+ super
9
+ @io_name = format('tcp://%s:%d', host, port)
10
+ @minion = master.spawn_minion do |ctx, counter|
11
+ socket = ctx.socket(:DEALER)
12
+ socket.identity = format('%d:%d', Process.pid, counter)
13
+ socket.connect(@io_name)
14
+ socket
15
+ end
16
+ end
17
+
18
+ def each
19
+ return enum_for(:each) unless block_given?
20
+ @minion.send('') # I'm ready!
21
+ while (msg = @minion.recv) != END_MSG
22
+ yield msg
23
+ @minion.send('') # More msg, pls!
24
+ end
25
+ end
26
+
27
+ def close
28
+ @minion.explode
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,30 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class SocketVentSendr < SocketChannel
4
+
5
+ attr_reader :io_name
6
+
7
+ def initialize(name, _config)
8
+ super
9
+ @io_name = format('tcp://*:%d', port)
10
+ @minion = master.spawn_minion { |ctx| ctx.bind(:ROUTER, @io_name) }
11
+ @receivers = Set.new
12
+ end
13
+
14
+ def <<(data)
15
+ identity = @minion.recv # Stop until ready
16
+ @receivers << identity
17
+ @minion.recv # Discard message ("command")
18
+ @minion.send_to(identity, data.to_s)
19
+ self
20
+ end
21
+
22
+ def close
23
+ @receivers.each do |identity|
24
+ @minion.send_to(identity, END_MSG)
25
+ end
26
+ @minion.explode
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,75 @@
1
+ require 'deadly_serious/engine/channel/socket/minion'
2
+ require 'deadly_serious/engine/channel/socket/master_mind'
3
+
4
+ module DeadlySerious
5
+ module Engine
6
+ class SocketChannel
7
+ # Odd, but I had too :(
8
+ require 'deadly_serious/engine/channel/socket/socket_vent_recvr'
9
+ require 'deadly_serious/engine/channel/socket/socket_vent_sendr'
10
+ require 'deadly_serious/engine/channel/socket/socket_sink_recvr'
11
+ require 'deadly_serious/engine/channel/socket/socket_sink_sendr'
12
+ include Enumerable
13
+
14
+ END_MSG = 'END TRANSMISSION'.freeze
15
+ RDY_MSG = 'READY FOR TRANSMISSION'.freeze
16
+
17
+ DEFAULT_PORT = 10001
18
+ REGEXP = /\A([<>][{}])([^:]+):(\d{1,5})\z/
19
+
20
+ attr_reader :host, :port, :master
21
+
22
+ def self.of_type(name)
23
+ matcher = name.match(REGEXP)
24
+ return if matcher.nil?
25
+ type = matcher[1]
26
+ case type
27
+ when '>{'
28
+ SocketVentSendr
29
+ when '<{'
30
+ SocketVentRecvr
31
+ when '>}'
32
+ SocketSinkSendr
33
+ when '<}'
34
+ SocketSinkRecvr
35
+ else
36
+ nil
37
+ end
38
+ end
39
+
40
+ def initialize(name, _config)
41
+ matcher = name.match(REGEXP)
42
+ host = matcher[2]
43
+ port = matcher[3]
44
+ @host = host.to_s.empty? ? 'localhost' : host.to_s
45
+ @port = port.to_s.empty? ? DEFAULT_PORT : port.to_i
46
+ @master = MasterMind.new_instance
47
+ end
48
+
49
+ def each
50
+ fail 'Subclass implementation'
51
+ end
52
+
53
+ def <<(_data)
54
+ fail 'Subclass implementation'
55
+ end
56
+
57
+ def close
58
+ fail 'Subclass implementation'
59
+ end
60
+
61
+ def flush
62
+ # Do nothing
63
+ end
64
+
65
+ def self.create(_name, _config)
66
+ # Do nothing
67
+ end
68
+
69
+ # Only for tests
70
+ def context
71
+ master.factory
72
+ end
73
+ end
74
+ end
75
+ end
@@ -2,6 +2,7 @@ module DeadlySerious
2
2
  module Engine
3
3
  # Commands make work with Pipelines easier.
4
4
  module Commands
5
+ MQUEUE_START_PORT = 13500
5
6
 
6
7
  private def auto_pipe
7
8
  @auto_pipe ||= AutoPipe.new
@@ -23,14 +24,14 @@ module DeadlySerious
23
24
  # the next component.
24
25
  def from_file(file_name, writer: next_pipe)
25
26
  file = file_name.sub(/^>?(.*)$/, '>\1')
26
- spawn_command('cat', reader: file, writer: writer)
27
+ spawn_command('cat ((<))', readers: [file], writers: [writer])
27
28
  end
28
29
 
29
30
  # Write a file to "data" dir from the pipe
30
31
  # of the last component
31
32
  def to_file(file_name, reader: last_pipe)
32
33
  file = file_name.sub(/^>?(.*)$/, '>\1')
33
- spawn_command('cat', reader: reader, writer: file)
34
+ spawn_command('cat', readers: [reader], writers: [file])
34
35
  end
35
36
 
36
37
  # Read from a specific named pipe.
@@ -38,7 +39,7 @@ module DeadlySerious
38
39
  # This is useful after a {#spawn_tee}, sometimes.
39
40
  def from_pipe(pipe_name, writer: next_pipe)
40
41
  pipe = pipe_name.sub(/^>?/, '')
41
- spawn_command('cat', reader: pipe, writer: writer)
42
+ spawn_command('cat', readers: [pipe], writers: [writer])
42
43
  end
43
44
 
44
45
  # Write the output of the last component to
@@ -49,7 +50,7 @@ module DeadlySerious
49
50
  # {#spawn_tee} instead.
50
51
  def to_pipe(pipe_name, reader: last_pipe)
51
52
  pipe = pipe_name.sub(/^>?/, '')
52
- spawn_command('cat', reader: reader, writer: pipe)
53
+ spawn_command('cat', readers: [reader], writers: [pipe])
53
54
  end
54
55
 
55
56
  # Spawn an object connected to the last and next components
@@ -65,6 +66,7 @@ module DeadlySerious
65
66
  # Spawn {number_of_processes} classes, one process for each of them.
66
67
  # Also, it divides the previous pipe in {number_of_processes} pipes,
67
68
  # an routes data through them.
69
+ # @deprecated
68
70
  def spawn_class_parallel(number_of_processes, class_name, *args, reader: last_pipe, writer: next_pipe)
69
71
  connect_a = (1..number_of_processes).map { |i| sprintf('%s.%da.splitter', class_name.to_s.downcase.gsub(/\W+/, '_'), i) }
70
72
  connect_b = (1..number_of_processes).map { |i| sprintf('%s.%db.splitter', class_name.to_s.downcase.gsub(/\W+/, '_'), i) }
@@ -75,8 +77,8 @@ module DeadlySerious
75
77
  spawn_process(DeadlySerious::Processes::Joiner, readers: connect_b, writers: [writer])
76
78
  end
77
79
 
78
- def spawn_lambda(reader: last_pipe, writer: next_pipe, &block)
79
- spawn_process(DeadlySerious::Processes::Lambda, block, readers: [reader], writers: [writer])
80
+ def spawn_lambda(name: 'Lambda',reader: last_pipe, writer: next_pipe, &block)
81
+ spawn_process(DeadlySerious::Processes::Lambda, block, process_name: name, readers: [reader], writers: [writer])
80
82
  end
81
83
 
82
84
  # Pipe from the last component to a intermediate
@@ -92,11 +94,13 @@ module DeadlySerious
92
94
 
93
95
  if block_given?
94
96
  on_subnet do
95
- spawn_command("tee #{create_pipe(next_pipe)}", reader: reader, writer: writer)
97
+ name = next_pipe
98
+ path = Channel.of_type(name).create(name, config)
99
+ spawn_command("tee #{path}", readers: [reader], writers: [writer])
96
100
  block.call
97
101
  end
98
102
  elsif escape
99
- spawn_command("tee #{create_pipe(escape)}", reader: reader, writer: writer)
103
+ spawn_command("tee #{Channel.of_type(escape).io_name_for(escape, config)}", readers: [reader], writers: [writer])
100
104
  else
101
105
  fail 'No block or escape given'
102
106
  end
@@ -114,6 +118,23 @@ module DeadlySerious
114
118
  w.create
115
119
  spawn("cat '#{r.io_name}' > '#{charger.io_name}' && cat '#{charger.io_name}' > '#{w.io_name}' && rm '#{charger.io_name}'")
116
120
  end
121
+
122
+ # Distribute data to "number_of_lanes" sub pipelines
123
+ def parallel(number_of_lanes, reader: last_pipe, writer: next_pipe)
124
+ @port ||= MQUEUE_START_PORT
125
+ ventilator = format('>{localhost:%d', @port)
126
+ input = format('<{localhost:%d', @port)
127
+ @port += 1
128
+ sink = format('<}localhost:%d', @port)
129
+ output = format('>}localhost:%d', @port)
130
+ @port += 1
131
+
132
+ spawn(Processes::Converter.new, reader: reader, writer: ventilator)
133
+ spawn(Processes::Converter.new, reader: sink, writer: writer)
134
+ on_subnet do
135
+ number_of_lanes.times { yield input, output }
136
+ end
137
+ end
117
138
  end
118
139
  end
119
140
  end
@@ -0,0 +1,55 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class Config
4
+ attr_reader :data_dir, :pipe_dir, :preserve_pipe_dir
5
+
6
+ def initialize(data_dir:, pipe_dir:, preserve_pipe_dir:)
7
+ @data_dir = data_dir
8
+ @pipe_dir = pipe_dir
9
+ @preserve_pipe_dir = preserve_pipe_dir
10
+ end
11
+
12
+ def file_path_for(name)
13
+ path_for(data_dir, name)
14
+ end
15
+
16
+ def pipe_path_for(name)
17
+ path_for(pipe_dir, name)
18
+ end
19
+
20
+ def path_for(directory, name)
21
+ if name =~ /^\//
22
+ # Absolute file path
23
+ name
24
+ else
25
+ # relative file path (relative to data_dir)
26
+ File.join(directory, name)
27
+ end
28
+ end
29
+
30
+ def setup
31
+ create_data_dir
32
+ create_pipe_dir
33
+ end
34
+
35
+ def teardown
36
+ destroy_pipe_dir
37
+ end
38
+
39
+ private
40
+
41
+ def create_data_dir
42
+ FileUtils.mkdir_p(@data_dir) unless File.exist?(@data_dir)
43
+ end
44
+
45
+ def create_pipe_dir
46
+ FileUtils.mkdir_p(@pipe_dir) unless File.exist?(@pipe_dir)
47
+ end
48
+
49
+ def destroy_pipe_dir
50
+ return if @preserve_pipe_dir || !File.exist?(@pipe_dir)
51
+ FileUtils.rm_r(@pipe_dir, force: true, secure: true)
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,57 @@
1
+ module DeadlySerious
2
+ module Engine
3
+ class FileMonitor
4
+ class Parts
5
+ attr_reader :directory, :name
6
+
7
+ def initialize(file_name)
8
+ matcher = file_name.to_s.match(%r{\A((?<dir>.*)/)?(?<name>[^/]+)\z})
9
+ @directory = matcher[:dir]
10
+ @name = matcher[:name]
11
+ end
12
+
13
+ def exist?
14
+ File.exist?(to_s)
15
+ end
16
+
17
+ def to_s
18
+ File.join(@directory, @name)
19
+ end
20
+ end
21
+
22
+ def initialize(*file_names)
23
+ @parts = file_names.map { |f| Parts.new(f) }
24
+ end
25
+
26
+ def wait_creation
27
+ part = @parts.find { |p| p.exist? }
28
+ return part.to_s if part
29
+ watch_event(@parts, :create)
30
+ end
31
+
32
+ def wait_modification
33
+ notifier = INotify::Notifier.new
34
+ @parts.each { |p| notifier.watch(p.to_s, :modify) { Fiber.yield p.to_s } }
35
+ fiber = Fiber.new { notifier.process }
36
+ fiber.resume
37
+ end
38
+
39
+ private
40
+
41
+ def watch_event(parts, event)
42
+ dirs = parts.group_by(&:directory)
43
+ notifier = INotify::Notifier.new
44
+ dirs.each do |dir, ps|
45
+ files = ps.map(&:name)
46
+ notifier.watch(dir, event) do |e|
47
+ Fiber.yield(File.join(dir, e.name)) if files.include?(e.name)
48
+ end
49
+ end
50
+ fiber = Fiber.new { notifier.run }
51
+ file_name = fiber.resume
52
+ notifier.stop
53
+ file_name
54
+ end
55
+ end
56
+ end
57
+ end
@@ -8,29 +8,31 @@ module DeadlySerious
8
8
  end
9
9
 
10
10
  def each
11
- if block_given?
12
- @io.each { |line| yield parse_line(line) }
13
- else
14
- @io.lazy.map { |line| parse_line(line) }
15
- end
16
- end
17
-
18
- def parse_line(line)
19
- MultiJson.load(line)
11
+ return enum_for(:each) unless block_given?
12
+ @io.each { |line| yield parse(line) }
20
13
  end
21
14
 
22
15
  def <<(value)
23
16
  case value
24
17
  when Hash
25
- @io << MultiJson.dump(value) << "\n"
18
+ @io << "#{MultiJson.dump(value)}\n"
26
19
  else
27
- @io << MultiJson.dump(Array(value)) << "\n"
20
+ @io << "#{MultiJson.dump(Array(value))}\n"
28
21
  end
29
22
  end
30
23
 
31
24
  def flush
32
25
  @io.flush
33
26
  end
27
+
28
+ private
29
+ def parse(line)
30
+ MultiJson.load(line)
31
+ rescue MultiJson::ParseError
32
+ puts 'Error in parse'
33
+ puts line
34
+ raise
35
+ end
34
36
  end
35
37
  end
36
38
  end