deadly_serious 1.0.2 → 2.0.0.pre.rc1

data/lib/deadly_serious/engine/pipeline.rb

@@ -3,28 +3,27 @@ module DeadlySerious
     class Pipeline
       include DeadlySerious::Engine::Commands
 
-      attr_reader :data_dir, :pipe_dir, :pids
+      attr_reader :pids, :config
 
       def initialize(data_dir: './data',
                      pipe_dir: "/tmp/deadly_serious/#{Process.pid}",
                      preserve_pipe_dir: false,
                      &block)
-        @data_dir = data_dir
-        @pipe_dir = pipe_dir
+
+        @config = Config.new(data_dir: data_dir, pipe_dir: pipe_dir, preserve_pipe_dir: preserve_pipe_dir)
         @block = block
         @pids = []
-        Channel.config(data_dir, pipe_dir, preserve_pipe_dir)
       end
 
       def run
-        Channel.setup
+        @config.setup
         @block.call(self)
         wait_children
       rescue => e
         kill_children
         raise e
       ensure
-        Channel.teardown
+        @config.teardown if @config
       end
 
       # Wait all sub processes to finish before
@@ -43,114 +42,59 @@ module DeadlySerious
       # prefer the simpler {DeadlySerious::Engine::Commands#spawn_class} or
       # the {DeadlySerious::Engine::Commands#spawn} methods.
       def spawn_process(class_or_object, *args, process_name: nil, readers: [last_pipe], writers: [next_pipe])
-        # TODO if we have no readers, alarm! (how about data sources???)
-        # TODO if we have no readers, and this is the first process, read from STDIN
-        # TODO if we have no writers, alarm! (how about data sinks???)
-        # TODO if we have no writers, and this is the last process, write to STDOUT
-        process_name ||= class_or_object.respond_to?(:name) ? class_or_object.name : class_or_object.to_s
-        writers.each { |writer| create_pipe(writer) }
+        writers.compact.each { |w| Channel.of_type(w).create(w, @config) }
         @pids << fork do
           begin
-            set_process_name(process_name, readers, writers)
-            # TODO Change this to not modify "a_class", so we can pass instances too
-            the_object = Class === class_or_object ? class_or_object.new : class_or_object
-            append_open_io_if_needed(the_object)
-            the_object.run(*args, readers: readers, writers: writers)
+            container = RubyObjectContainer.new(class_or_object,
+                                                args,
+                                                process_name,
+                                                @config,
+                                                readers.compact,
+                                                writers.compact)
+            set_process_name(container.name)
+            container.run
           rescue Errno::EPIPE # Broken Pipe, no problem
             # Ignore
           ensure
-            the_object.finalize if the_object.respond_to?(:finalize)
+            container.finalize if container
           end
         end
       end
 
-      def spawn_command(a_shell_command, env: {}, reader: nil, writer: nil, readers: [], writers: [])
-        input_pattern = '((<))'
-        output_pattern = '((>))'
-
-        if reader.nil? && readers.empty?
-          readers << last_pipe
-        elsif reader && readers.empty?
-          readers << reader
-        end
-
-        if writer.nil? && writers.empty?
-          writers << next_pipe
-        elsif writer && writers.empty?
-          writers << writer
-        end
-
-
-        shell_tokens = case a_shell_command
-                         when Array
-                           a_shell_command
-                         else
-                           a_shell_command.to_s.split(/\s+/)
-                       end
-
-        inputs = readers.map { |it| create_pipe(it) }
-        outputs = writers.map { |it| create_pipe(it) }
-
-        tokens = shell_tokens.map do |token|
-          case token
-            when input_pattern
-              inputs.shift || fail('Missing reader')
-            when output_pattern
-              outputs.shift || fail('Missing writer')
-            else
-              token.to_s
+      def spawn_command(a_shell_command, env: {}, readers: [last_pipe], writers: [next_pipe])
+        writers.compact.each { |w| Channel.of_type(w).create(w, @config) }
+        @pids << fork do
+          begin
+            container = SoCommandContainer.new(a_shell_command,
+                                               env,
+                                               @config,
+                                               readers.compact,
+                                               writers.compact)
+            set_process_name(container.name)
+            container.run
+          rescue Errno::EPIPE # Broken Pipe, no problem
+            # Ignore
           end
         end
-
-        in_out = {close_others: true,
-                  in: inputs.size == 1 ? [inputs.first, 'r'] : :close,
-                  out: outputs.size == 1 ? [outputs.first, 'w'] : :close}
-
-        description = "#{tokens.first} #{in_out}"
-        @pids << fork { exec(env, [tokens.first, description], *tokens[1..-1], in_out) }
       end
 
       private
 
-      def append_open_io_if_needed(an_object)
-        class << an_object
-          prepend OpenIo
-        end
-      end
-
-      def create_pipe(pipe_name)
-        Channel.create_pipe(pipe_name)
-      end
-
       def wait_children
-        # Don't wait if we have no children
-        # Thread.start do
-        #   while @pids.size > 1
-        #     @pids.each do |pid|
-        #       begin
-        #         Process.wait(pid, Process::WNOHANG)
-        #       rescue Errno::ECHILD
-        #         puts "Try delete #{pid}"
-        #         @pids.delete(pid)
-        #       end
-        #     end
-        #     sleep(0.5)
-        #   end
-        # end
         Process.waitall
-        @pids.clear
       end
 
       def kill_children
         gpid = Process.gid
         Process.kill('SIGTERM', -gpid) rescue nil
         Timeout::timeout(5) { wait_children }
+        @pids.clear
       rescue Timeout::Error
         Process.kill('SIGKILL', -gpid) rescue nil
       end
 
-      def set_process_name(name, readers, writers)
-        $0 = format('(%s)-->[%s]-->(%s)', readers.join(', '), name, writers.join(' '))
+      def set_process_name(name)
+        $0 = name
       end
     end
   end

data/lib/deadly_serious/engine/ruby_object_container.rb

@@ -0,0 +1,42 @@
+module DeadlySerious
+  module Engine
+    class RubyObjectContainer
+      attr_reader :name
+
+      def initialize(class_or_object, args, process_name, config, reader_names, writers_names)
+        @args = args
+        @config = config
+        @reader_names = reader_names
+        @writer_names = writers_names
+
+        @the_object = prepare_object(class_or_object)
+        @name = prepare_process_name(@the_object, process_name, reader_names, writers_names)
+      end
+
+      def run
+        readers = @reader_names.map { |r| Channel.new(r, @config) }
+        writers = @writer_names.map { |w| Channel.new(w, @config) }
+        @the_object.run(*@args, readers: readers, writers: writers)
+      ensure
+        writers.each { |w| w.close if w } if writers
+        readers.each { |r| r.close if r } if readers
+      end
+
+      def finalize
+        @the_object.finalize if @the_object.respond_to?(:finalize)
+      end
+
+      private
+
+      def prepare_object(class_or_object)
+        Class === class_or_object ? class_or_object.new : class_or_object
+      end
+
+      def prepare_process_name(the_object, process_name, reader_names, writer_names)
+        return process_name if process_name
+        name = the_object.respond_to?(:name) ? the_object.name : the_object.to_s
+        format('(%s)-->[%s]-->(%s)', reader_names.join(' '), name, writer_names.join(' '))
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/so_command_container.rb

@@ -0,0 +1,56 @@
+module DeadlySerious
+  module Engine
+    class SoCommandContainer
+      INPUT_PATTERN = '((<))'
+      OUTPUT_PATTERN = '((>))'
+      attr_reader :name
+
+      def initialize(a_shell_command, env, config, reader_names, writer_names)
+        @env = env
+        @readers = reader_names.map { |r| Channel.of_type(r).io_name_for(r, config) }
+        @writers = writer_names.map { |w| Channel.of_type(w).io_name_for(w, config) }
+        @tokens = prepare_command(a_shell_command)
+        @name = @tokens.first +
+            @readers.map { |it| " <#{it}" }.join('') +
+            @writers.map { |it| " >#{it}" }.join('')
+      end
+
+      def run
+        in_out = {close_others: true,
+                  in: @readers.size == 1 ? [@readers.first, 'r'] : :close,
+                  out: @writers.size == 1 ? [@writers.first, 'w'] : :close}
+
+        exec(@env, [@tokens.first, name], *@tokens[1..-1], in_out)
+      end
+
+      private
+
+      def prepare_command(a_shell_command)
+        shell_tokens = shell_tokens(a_shell_command)
+        replace_placeholders(shell_tokens)
+      end
+
+      def shell_tokens(a_shell_command)
+        case a_shell_command
+          when Array
+            a_shell_command
+          else
+            a_shell_command.to_s.split(/\s+/)
+        end
+      end
+
+      def replace_placeholders(shell_tokens)
+        shell_tokens.map do |token|
+          case token
+            when INPUT_PATTERN
+              @readers.shift || fail('Missing reader')
+            when OUTPUT_PATTERN
+              @writers.shift || fail('Missing writer')
+            else
+              token.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deadly_serious/processes/converter.rb

@@ -0,0 +1,12 @@
+module DeadlySerious
+  module Processes
+    class Converter
+      def run(readers: [], writers: [])
+        reader = readers.first
+        reader.each do |line|
+          writers.each { |w| w << line }
+        end
+      end
+    end
+  end
+end

data/lib/deadly_serious/processes/lambda.rb

@@ -6,8 +6,8 @@ module DeadlySerious
         writer_param = params.any? { |(k, n)| k == :keyreq && n == :writer }
         reader_param = params.any? { |(k, n)| k == :keyreq && n == :reader }
 
-        reader = JsonIo.new(readers.first) if readers.size == 1 && File.exist?(readers.first.filename)
-        writer = JsonIo.new(writers.first) if writers.size == 1 && File.exist?(writers.first.filename)
+        reader = JsonIo.new(readers.first) if readers.size == 1
+        writer = JsonIo.new(writers.first) if writers.size == 1
 
         if reader_param && writer_param
           unless reader
@@ -25,6 +25,8 @@ module DeadlySerious
           # however, it's awesomely useful. =\
           reader.each do |data|
             result = block.call(*data)
+
+            # noinspection RubySimplifyBooleanInspection
             if result == true # really TRUE, not thruthy
               # Acts as filter
               writer << data

data/lib/deadly_serious/processes/resilient_splitter.rb

@@ -19,7 +19,7 @@ module DeadlySerious
               outputs = writers.dup
             end
             current = outputs.first
-            current << line
+            current << line << "\n"
             outputs.rotate!
           rescue Errno::EPIPE => e
             puts e.inspect

data/lib/deadly_serious/version.rb

@@ -1,3 +1,3 @@
 module DeadlySerious
-  VERSION = '1.0.2'
+  VERSION = '2.0.0-rc1'
 end

data/spec/lib/deadly_serious/engine/auto_pipe_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+include DeadlySerious::Engine
+
+describe AutoPipe do
+  subject { AutoPipe.new }
+  describe '#last' do
+    it 'returns last writer' do
+      subject.next
+      expect(subject.last).to eq 'pipe.0001'
+    end
+
+    it 'returns nil if no last writer' do
+      expect(subject.last).to be_nil
+    end
+  end
+
+  describe '#next' do
+    it 'returns next writer name' do
+      expect(subject.next).to eq 'pipe.0001'
+      expect(subject.next).to eq 'pipe.0002'
+    end
+  end
+
+  describe '#on_subnet' do
+    it 'creates "subnames" to avoid conflicts' do
+      subject.next
+      subject.on_subnet do
+        expect(subject.next).to eq 'pipe.0001.0001'
+        expect(subject.last).to eq 'pipe.0001.0001'
+        expect(subject.next).to eq 'pipe.0001.0002'
+        subject.on_subnet do
+          expect(subject.next).to eq 'pipe.0001.0002.0001'
+        end
+      end
+      subject.next
+      subject.on_subnet do
+        expect(subject.next).to eq 'pipe.0002.0001'
+      end
+    end
+  end
+end

data/spec/lib/deadly_serious/engine/channel/socket_channel_spec.rb

@@ -0,0 +1,159 @@
+require 'spec_helper'
+include DeadlySerious::Engine
+
+describe SocketChannel do
+  let(:test_config) do
+    Config.new(data_dir: "/tmp/test_deadly_serious#{Process.pid}",
+               pipe_dir: "/tmp/test_deadly_serious#{Process.pid}",
+               preserve_pipe_dir: false)
+  end
+
+  matcher :produce_a do |expected|
+    subject = nil
+    match do |actual|
+      begin
+        subject = Channel.new(actual, test_config)
+        subject.is_a?(expected)
+      ensure
+        subject.close if subject
+      end
+    end
+    failure_message do
+      "but was a #{subject.class.name}"
+    end
+  end
+
+  describe 'sink' do
+    let(:snd) { '>}' }
+    let(:rcv) { '<}' }
+
+    def send_msg(*msgs, port: 5555)
+      fork do
+        sender = Channel.new("#{snd}localhost:#{port}", nil)
+        msgs.each { |m| sender << m }
+        sender.close
+      end
+    end
+
+    it 'detects sender by its name' do
+      expect("#{snd}localhost:5555").to produce_a SocketSinkSendr
+    end
+
+    it 'detects receiver by its name' do
+      expect("#{rcv}localhost:5555").to produce_a SocketSinkRecvr
+    end
+
+    it 'connect two simple processes' do
+      begin
+        send_msg(1, 2)
+        receiver = Channel.new("#{rcv}localhost:5555", nil)
+        stream = receiver.each
+        expect(stream.next).to eq '1'
+        expect(stream.next).to eq '2'
+      ensure
+        receiver.close if receiver
+      end
+    end
+  end
+
+  describe 'ventilator' do
+    let(:snd) { '>{' }
+    let(:rcv) { '<{' }
+
+    def send_msg(*msgs, port: 5555)
+      fork do
+        sender = Channel.new("#{snd}localhost:#{port}", nil)
+        msgs.each { |m| sender << m }
+        sender.close
+      end
+    end
+
+    it 'detects sender by its name' do
+      expect("#{snd}localhost:5555").to produce_a SocketVentSendr
+    end
+
+    it 'detects receiver by its name' do
+      expect("#{rcv}localhost:5555").to produce_a SocketVentRecvr
+    end
+
+    it 'connect two simple processes' do
+      begin
+        send_msg(1, 2)
+        receiver = Channel.new("#{rcv}localhost:5555", nil)
+        stream = receiver.each
+        expect(stream.next).to eq '1'
+        expect(stream.next).to eq '2'
+      ensure
+        receiver.close if receiver
+      end
+    end
+
+    it 'has the same ZMQ context for multiple channels' do
+      c1 = Channel.new("#{rcv}localhost:5555", nil)
+      ctx1 = c1.context
+
+      c2 = Channel.new("#{snd}localhost:5556", nil)
+      ctx2 = c2.context
+
+      expect(ctx1).to eq ctx2
+      c1.close
+      c2.close
+    end
+
+    it 'has different contexts when open/close twice' do
+      c1 = Channel.new("#{rcv}localhost:5555", nil)
+      ctx1 = c1.context
+      c1.close
+
+      c2 = Channel.new("#{snd}localhost:5556", nil)
+      ctx2 = c2.context
+      c2.close
+
+      expect(ctx1).not_to eq ctx2
+    end
+
+    it 'clear ZMQ context on finish a single channel' do
+      send_msg(1)
+      channel = Channel.new("#{rcv}localhost:5555", nil)
+      channel.each {}
+      ctx = channel.context
+      channel.close
+      expect do
+        ctx.bind(:PUSH, 'tcp://*:5556')
+      end.to raise_error(ZMQ::Error, /has been destroyed/)
+    end
+
+    it 'clear ZMQ context on finish all channels' do
+      channel1 = Channel.new("#{rcv}localhost:5555", nil)
+      channel2 = Channel.new("#{rcv}localhost:5556", nil)
+
+      ctx = channel1.context
+
+      channel1.close
+      expect do
+        ctx.bind(:PUSH, 'tcp://*:5557')
+      end.not_to raise_error
+
+      channel2.close
+      expect do
+        ctx.bind(:PUSH, 'tcp://*:5558')
+      end.to raise_error(ZMQ::Error, /has been destroyed/)
+    end
+
+    it 'does load balancing' do
+      send_msg(1, 2, 3)
+      channel1 = Channel.new("#{rcv}localhost:5555", nil)
+      channel2 = Channel.new("#{rcv}localhost:5555", nil)
+
+      c1 = channel1.each
+      c2 = channel2.each
+
+      expect(c1.next).to eq '1'
+      expect(c2.next).to eq '2'
+      expect(c1.next).to eq '3'
+
+      channel1.close
+      channel2.close
+    end
+  end
+end