deadly_serious 1.0.2 → 2.0.0.pre.rc1

data/lib/deadly_serious/engine/channel/socket/socket_sink_recvr.rb

@@ -0,0 +1,34 @@
+module DeadlySerious
+  module Engine
+    class SocketSinkRecvr < SocketChannel
+
+      attr_reader :io_name
+
+      def initialize(name, _config)
+        super
+        @io_name = format('tcp://*:%d', port)
+        @minion = master.spawn_minion { |ctx| ctx.bind(:PULL, @io_name) }
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+        clients = 0
+        loop do
+          msg = @minion.recv
+          if msg == END_MSG
+            clients -= 1
+            break if clients <= 0
+          elsif msg == RDY_MSG
+            clients += 1
+          else
+            yield msg
+          end
+        end
+      end
+
+      def close
+        @minion.explode
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/channel/socket/socket_sink_sendr.rb

@@ -0,0 +1,26 @@
+module DeadlySerious
+  module Engine
+    class SocketSinkSendr < SocketChannel
+
+      attr_reader :io_name
+
+      def initialize(name, _config)
+        super
+        @io_name = format('tcp://%s:%d', host, port)
+        @minion = master.spawn_minion { |ctx| ctx.connect(:PUSH, @io_name) }
+        sleep(0.5) # Avoid slow joiner syndrome the stupid way >(
+        @minion.send(RDY_MSG)
+      end
+
+      def <<(data)
+        @minion.send(data.to_s)
+        self
+      end
+
+      def close
+        @minion.send(END_MSG)
+        @minion.explode
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/channel/socket/socket_vent_recvr.rb

@@ -0,0 +1,32 @@
+module DeadlySerious
+  module Engine
+    class SocketVentRecvr < SocketChannel
+
+      attr_reader :io_name
+
+      def initialize(name, _config)
+        super
+        @io_name = format('tcp://%s:%d', host, port)
+        @minion = master.spawn_minion do |ctx, counter|
+          socket = ctx.socket(:DEALER)
+          socket.identity = format('%d:%d', Process.pid, counter)
+          socket.connect(@io_name)
+          socket
+        end
+      end
+
+      def each
+        return enum_for(:each) unless block_given?
+        @minion.send('') # I'm ready!
+        while (msg = @minion.recv) != END_MSG
+          yield msg
+          @minion.send('') # More msg, pls!
+        end
+      end
+
+      def close
+        @minion.explode
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/channel/socket/socket_vent_sendr.rb

@@ -0,0 +1,30 @@
+module DeadlySerious
+  module Engine
+    class SocketVentSendr < SocketChannel
+
+      attr_reader :io_name
+
+      def initialize(name, _config)
+        super
+        @io_name = format('tcp://*:%d', port)
+        @minion = master.spawn_minion { |ctx| ctx.bind(:ROUTER, @io_name) }
+        @receivers = Set.new
+      end
+
+      def <<(data)
+        identity = @minion.recv # Stop until ready
+        @receivers << identity
+        @minion.recv # Discard message ("command")
+        @minion.send_to(identity, data.to_s)
+        self
+      end
+
+      def close
+        @receivers.each do |identity|
+          @minion.send_to(identity, END_MSG)
+        end
+        @minion.explode
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/channel/socket_channel.rb

@@ -0,0 +1,75 @@
+require 'deadly_serious/engine/channel/socket/minion'
+require 'deadly_serious/engine/channel/socket/master_mind'
+
+module DeadlySerious
+  module Engine
+    class SocketChannel
+      # Odd, but I had too :(
+      require 'deadly_serious/engine/channel/socket/socket_vent_recvr'
+      require 'deadly_serious/engine/channel/socket/socket_vent_sendr'
+      require 'deadly_serious/engine/channel/socket/socket_sink_recvr'
+      require 'deadly_serious/engine/channel/socket/socket_sink_sendr'
+      include Enumerable
+
+      END_MSG = 'END TRANSMISSION'.freeze
+      RDY_MSG = 'READY FOR TRANSMISSION'.freeze
+
+      DEFAULT_PORT = 10001
+      REGEXP = /\A([<>][{}])([^:]+):(\d{1,5})\z/
+
+      attr_reader :host, :port, :master
+
+      def self.of_type(name)
+        matcher = name.match(REGEXP)
+        return if matcher.nil?
+        type = matcher[1]
+        case type
+          when '>{'
+            SocketVentSendr
+          when '<{'
+            SocketVentRecvr
+          when '>}'
+            SocketSinkSendr
+          when '<}'
+            SocketSinkRecvr
+          else
+            nil
+        end
+      end
+
+      def initialize(name, _config)
+        matcher = name.match(REGEXP)
+        host = matcher[2]
+        port = matcher[3]
+        @host = host.to_s.empty? ? 'localhost' : host.to_s
+        @port = port.to_s.empty? ? DEFAULT_PORT : port.to_i
+        @master = MasterMind.new_instance
+      end
+
+      def each
+        fail 'Subclass implementation'
+      end
+
+      def <<(_data)
+        fail 'Subclass implementation'
+      end
+
+      def close
+        fail 'Subclass implementation'
+      end
+
+      def flush
+        # Do nothing
+      end
+
+      def self.create(_name, _config)
+        # Do nothing
+      end
+
+      # Only for tests
+      def context
+        master.factory
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/commands.rb

@@ -2,6 +2,7 @@ module DeadlySerious
   module Engine
     # Commands make work with Pipelines easier.
     module Commands
+      MQUEUE_START_PORT = 13500
 
       private def auto_pipe
         @auto_pipe ||= AutoPipe.new
@@ -23,14 +24,14 @@ module DeadlySerious
       # the next component.
       def from_file(file_name, writer: next_pipe)
         file = file_name.sub(/^>?(.*)$/, '>\1')
-        spawn_command('cat', reader: file, writer: writer)
+        spawn_command('cat ((<))', readers: [file], writers: [writer])
       end
 
       # Write a file to "data" dir from the pipe
       # of the last component
       def to_file(file_name, reader: last_pipe)
         file = file_name.sub(/^>?(.*)$/, '>\1')
-        spawn_command('cat', reader: reader, writer: file)
+        spawn_command('cat', readers: [reader], writers: [file])
       end
 
       # Read from a specific named pipe.
@@ -38,7 +39,7 @@ module DeadlySerious
       # This is useful after a {#spawn_tee}, sometimes.
       def from_pipe(pipe_name, writer: next_pipe)
         pipe = pipe_name.sub(/^>?/, '')
-        spawn_command('cat', reader: pipe, writer: writer)
+        spawn_command('cat', readers: [pipe], writers: [writer])
       end
 
       # Write the output of the last component to
@@ -49,7 +50,7 @@ module DeadlySerious
       # {#spawn_tee} instead.
       def to_pipe(pipe_name, reader: last_pipe)
         pipe = pipe_name.sub(/^>?/, '')
-        spawn_command('cat', reader: reader, writer: pipe)
+        spawn_command('cat', readers: [reader], writers: [pipe])
       end
 
       # Spawn an object connected to the last and next components
@@ -65,6 +66,7 @@ module DeadlySerious
       # Spawn {number_of_processes} classes, one process for each of them.
       # Also, it divides the previous pipe in {number_of_processes} pipes,
       # an routes data through them.
+      # @deprecated
       def spawn_class_parallel(number_of_processes, class_name, *args, reader: last_pipe, writer: next_pipe)
         connect_a = (1..number_of_processes).map { |i| sprintf('%s.%da.splitter', class_name.to_s.downcase.gsub(/\W+/, '_'), i) }
         connect_b = (1..number_of_processes).map { |i| sprintf('%s.%db.splitter', class_name.to_s.downcase.gsub(/\W+/, '_'), i) }
@@ -75,8 +77,8 @@ module DeadlySerious
         spawn_process(DeadlySerious::Processes::Joiner, readers: connect_b, writers: [writer])
       end
 
-      def spawn_lambda(reader: last_pipe, writer: next_pipe, &block)
-        spawn_process(DeadlySerious::Processes::Lambda, block, readers: [reader], writers: [writer])
+      def spawn_lambda(name: 'Lambda',reader: last_pipe, writer: next_pipe, &block)
+        spawn_process(DeadlySerious::Processes::Lambda, block, process_name: name, readers: [reader], writers: [writer])
       end
 
       # Pipe from the last component to a intermediate
@@ -92,11 +94,13 @@ module DeadlySerious
 
         if block_given?
           on_subnet do
-            spawn_command("tee #{create_pipe(next_pipe)}", reader: reader, writer: writer)
+            name = next_pipe
+            path = Channel.of_type(name).create(name, config)
+            spawn_command("tee #{path}", readers: [reader], writers: [writer])
             block.call
           end
         elsif escape
-          spawn_command("tee #{create_pipe(escape)}", reader: reader, writer: writer)
+          spawn_command("tee #{Channel.of_type(escape).io_name_for(escape, config)}", readers: [reader], writers: [writer])
         else
           fail 'No block or escape given'
         end
@@ -114,6 +118,23 @@ module DeadlySerious
         w.create
         spawn("cat '#{r.io_name}' > '#{charger.io_name}' && cat '#{charger.io_name}' > '#{w.io_name}' && rm '#{charger.io_name}'")
       end
+
+      # Distribute data to "number_of_lanes" sub pipelines
+      def parallel(number_of_lanes, reader: last_pipe, writer: next_pipe)
+        @port ||= MQUEUE_START_PORT
+        ventilator = format('>{localhost:%d', @port)
+        input = format('<{localhost:%d', @port)
+        @port += 1
+        sink = format('<}localhost:%d', @port)
+        output = format('>}localhost:%d', @port)
+        @port += 1
+
+        spawn(Processes::Converter.new, reader: reader, writer: ventilator)
+        spawn(Processes::Converter.new, reader: sink, writer: writer)
+        on_subnet do
+          number_of_lanes.times { yield input, output }
+        end
+      end
     end
   end
 end

data/lib/deadly_serious/engine/config.rb

@@ -0,0 +1,55 @@
+module DeadlySerious
+  module Engine
+    class Config
+      attr_reader :data_dir, :pipe_dir, :preserve_pipe_dir
+
+      def initialize(data_dir:, pipe_dir:, preserve_pipe_dir:)
+        @data_dir = data_dir
+        @pipe_dir = pipe_dir
+        @preserve_pipe_dir = preserve_pipe_dir
+      end
+
+      def file_path_for(name)
+        path_for(data_dir, name)
+      end
+
+      def pipe_path_for(name)
+        path_for(pipe_dir, name)
+      end
+
+      def path_for(directory, name)
+        if name =~ /^\//
+          # Absolute file path
+          name
+        else
+          # relative file path (relative to data_dir)
+          File.join(directory, name)
+        end
+      end
+
+      def setup
+        create_data_dir
+        create_pipe_dir
+      end
+
+      def teardown
+        destroy_pipe_dir
+      end
+
+      private
+
+      def create_data_dir
+        FileUtils.mkdir_p(@data_dir) unless File.exist?(@data_dir)
+      end
+
+      def create_pipe_dir
+        FileUtils.mkdir_p(@pipe_dir) unless File.exist?(@pipe_dir)
+      end
+
+      def destroy_pipe_dir
+        return if @preserve_pipe_dir || !File.exist?(@pipe_dir)
+        FileUtils.rm_r(@pipe_dir, force: true, secure: true)
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/file_monitor.rb

@@ -0,0 +1,57 @@
+module DeadlySerious
+  module Engine
+    class FileMonitor
+      class Parts
+        attr_reader :directory, :name
+
+        def initialize(file_name)
+          matcher = file_name.to_s.match(%r{\A((?<dir>.*)/)?(?<name>[^/]+)\z})
+          @directory = matcher[:dir]
+          @name = matcher[:name]
+        end
+
+        def exist?
+          File.exist?(to_s)
+        end
+
+        def to_s
+          File.join(@directory, @name)
+        end
+      end
+
+      def initialize(*file_names)
+        @parts = file_names.map { |f| Parts.new(f) }
+      end
+
+      def wait_creation
+        part = @parts.find { |p| p.exist? }
+        return part.to_s if part
+        watch_event(@parts, :create)
+      end
+
+      def wait_modification
+        notifier = INotify::Notifier.new
+        @parts.each { |p| notifier.watch(p.to_s, :modify) { Fiber.yield p.to_s } }
+        fiber = Fiber.new { notifier.process }
+        fiber.resume
+      end
+
+      private
+
+      def watch_event(parts, event)
+        dirs = parts.group_by(&:directory)
+        notifier = INotify::Notifier.new
+        dirs.each do |dir, ps|
+          files = ps.map(&:name)
+          notifier.watch(dir, event) do |e|
+            Fiber.yield(File.join(dir, e.name)) if files.include?(e.name)
+          end
+        end
+        fiber = Fiber.new { notifier.run }
+        file_name = fiber.resume
+        notifier.stop
+        file_name
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/json_io.rb

@@ -8,29 +8,31 @@ module DeadlySerious
       end
 
       def each
-        if block_given?
-          @io.each { |line| yield parse_line(line) }
-        else
-          @io.lazy.map { |line| parse_line(line) }
-        end
-      end
-
-      def parse_line(line)
-        MultiJson.load(line)
+        return enum_for(:each) unless block_given?
+        @io.each { |line| yield parse(line) }
       end
 
       def <<(value)
         case value
           when Hash
-            @io << MultiJson.dump(value) << "\n"
+            @io << "#{MultiJson.dump(value)}\n"
           else
-            @io << MultiJson.dump(Array(value)) << "\n"
+            @io << "#{MultiJson.dump(Array(value))}\n"
         end
       end
 
       def flush
         @io.flush
       end
+
+      private
+      def parse(line)
+        MultiJson.load(line)
+      rescue MultiJson::ParseError
+        puts 'Error in parse'
+        puts line
+        raise
+      end
     end
   end
 end