deadly_serious 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 775816edf4860d7516908442da4d8ffaa2793aba
-  data.tar.gz: 0516bbdbccd02bf0102833c1dfcbe2ecc1bbac9f
+  metadata.gz: c9050728b34b8dd157c703876d8e5fe9e60a8453
+  data.tar.gz: 613441848d930e6429eac0660040f795ce868f94
 SHA512:
-  metadata.gz: 9ddb27632bffcbba2bd7c9f008e0662b27c5dcdcd048ae244cffbe577f7afdcfb19c5c7723891dde7f221f9cf81ccd7681d0aec0ebc9e62d6f25c7cf3820ec7a
-  data.tar.gz: d49e4442d46a296c638d84e6120ae83d25f174d55e353c34e2a63a9741a5116b49f738be2a8642356f84932fa16872884b0a01ec66ca233013a62173adecd609
+  metadata.gz: 4294cc634cc2b4b417f4f0dff25f7a8a706d5ef1e262505d0dcf14f8a4834cf9ce22e19c906bd04c5bcf27792a5b8045ccf5e0c18a7e68a77d461af35da99064
+  data.tar.gz: d7f9fd19630d96c51b79524b6d0c2995023aad7b1ca0d9df49e96e18953ab856e4f43e961bb913cde0b7cc868bcdcae4016f3f654c45816e3605983e26c33b2d

data/.gitignore

@@ -1,18 +1,20 @@
-*.gem
-*.rbc
 .bundle
 .config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
+.idea/
 coverage
 doc/
+*.gem
+Gemfile.lock
+InstalledFiles
 lib/bundler/man
+output.data
 pkg
+*.rbc
 rdoc
 spec/reports
+tags
 test/tmp
 test/version_tmp
 tmp
-output.data
+_yardoc
+.yardoc

data/deadly_serious.gemspec

@@ -20,7 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'yard'
 
   spec.add_dependency 'json'
 end

data/lib/deadly_serious/engine/base_process.rb

@@ -8,16 +8,27 @@ module DeadlySerious
         reader.each { |packet| super(packet.chomp) }
       end
 
+      # Alias to #send
+      def emit(packet = nil)
+        send(packet)
+      end
+
+      # Send a packet to the next process
       def send(packet = nil)
         send_buffered(packet)
         flush_buffer
       end
 
+      # Send a packet to the next process,
+      # however, accumulate some of them
+      # before send to gain a little
+      # efficency.
       def send_buffered(packet = nil)
         @writer << packet if packet
         @writer << "\n"
       end
 
+      # Send all not yet sent packets.
       def flush_buffer
         @writer.flush
       end

data/lib/deadly_serious/engine/channel.rb

@@ -1,19 +1,40 @@
 require 'socket'
+require 'deadly_serious/engine/lazy_io'
 
 module DeadlySerious
   module Engine
     # Fake class, it's actually a factory ¬¬
     module Channel
-      def self.new(name, data_dir: nil, pipe_dir: nil)
+      def self.new(name)
         matcher = name.match(/^(>)?(.*?)(?:(:)(\d{1,5}))?$/)
         if matcher[1] == '>'
-          FileChannel.new(matcher[2], data_dir)
+          FileChannel.new(matcher[2], @data_dir)
         elsif matcher[3] == ':'
           SocketChannel.new(matcher[2], matcher[4].to_i)
         else
-          PipeChannel.new(matcher[2], pipe_dir)
+          PipeChannel.new(matcher[2], @pipe_dir)
         end
       end
+
+      def self.config(data_dir, pipe_dir, preserve_pipe_dir)
+        @data_dir = data_dir
+        @pipe_dir = pipe_dir
+        @preserve_pipe_dir = preserve_pipe_dir
+      end
+
+      def self.setup
+        FileUtils.mkdir_p(@pipe_dir) unless File.exist?(@pipe_dir)
+      end
+
+      def self.teardown
+        if !@preserve_pipe_dir && File.exist?(@pipe_dir)
+          FileUtils.rm_r(@pipe_dir, force: true, secure: true)
+        end
+      end
+
+      def self.create_pipe(pipe_name)
+        new(pipe_name).create
+      end
     end
 
     class FileChannel
@@ -35,6 +56,10 @@ module DeadlySerious
         fail %(File "#{@io_name}" not found) unless File.exist?(@io_name)
         open(@io_name, 'w')
       end
+
+      def io
+        LazyIo.new(self)
+      end
     end
 
     class PipeChannel
@@ -56,6 +81,10 @@ module DeadlySerious
         fail %(Pipe "#{@io_name}" not found) unless File.exist?(@io_name)
         open(@io_name, 'w')
       end
+
+      def io
+        LazyIo.new(self)
+      end
     end
 
     class SocketChannel
@@ -83,6 +112,10 @@ module DeadlySerious
         server = TCPServer.new(@port)
         server.accept
       end
+
+      def io
+        LazyIo.new(self)
+      end
     end
   end
 end

data/lib/deadly_serious/engine/lazy_io.rb

@@ -0,0 +1,55 @@
+module DeadlySerious
+  module Engine
+
+    # Restrict IO class that opens ONLY
+    # when trying to read something.
+    #
+    # Also, used to reopend lost connections.
+    #
+    # By "restrict", I mean it implements
+    # just a few IO operations.
+    class LazyIo
+      def initialize(channel)
+        @channel = channel
+      end
+
+      def gets
+        open_reader
+        @io.gets
+      end
+
+      def each(&block)
+        open_reader
+        @io.each &block
+      end
+
+      def <<(element)
+        open_writer
+        @io << element
+      end
+
+      def closed?
+        @io.nil? || @io.closed?
+      end
+
+      def close
+        @io.close
+        @io = nil
+      end
+
+      private
+
+      def open_reader
+        if closed?
+          @io = @channel.open_reader
+        end
+      end
+
+      def open_writer
+        if closed?
+          @io = @channel.open_writer
+        end
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/open_io.rb

@@ -0,0 +1,37 @@
+require 'fileutils'
+
+module DeadlySerious
+  module Engine
+    module OpenIo
+      def run(*args, readers: [], writers:[])
+        opened_readers = readers.map { |reader| wrap_io(reader) }
+        opened_writers = writers.map { |writer| wrap_io(writer) }
+        super(*args, readers: opened_readers, writers: opened_writers)
+      ensure
+        if opened_writers
+          opened_writers.each { |writer| close_io(writer) }
+        end
+        if opened_readers
+          opened_readers.each { |reader| close_io(reader) }
+        end
+      end
+
+      private
+
+      def close_io(io)
+        return unless io
+        return if io.closed?
+        io.close
+      rescue => e
+        # Intentionally eat the error
+        # because it's being used inside
+        # an "ensure" block
+        puts e.inspect
+      end
+
+      def wrap_io(pipe_name)
+        Channel.new(pipe_name).io
+      end
+    end
+  end
+end

data/lib/deadly_serious/engine/spawner.rb

@@ -1,55 +1,75 @@
-require 'fileutils'
 require 'deadly_serious/engine/channel'
+require 'deadly_serious/engine/open_io'
+require 'deadly_serious/processes/splitter'
 
 module DeadlySerious
   module Engine
     class Spawner
-      def initialize(data_dir: './data', pipe_dir: "/tmp/deadly_serious/#{Process.pid}", preserve_pipe_dir: false)
-        @data_dir = data_dir
-        @pipe_dir = pipe_dir
-        @preserve_pipe_dir = preserve_pipe_dir
+      def initialize(data_dir: './data',
+                     pipe_dir: "/tmp/deadly_serious/#{Process.pid}",
+                     preserve_pipe_dir: false)
         @ids = []
-
-        FileUtils.mkdir_p(pipe_dir) unless File.exist?(pipe_dir)
+        Channel.config(data_dir, pipe_dir, preserve_pipe_dir)
       end
 
       def run
+        Channel.setup
         run_pipeline
         wait_children
-      rescue Exception => e
+      rescue => e
         kill_children
         raise e
       ensure
-        if !@preserve_pipe_dir && File.exist?(@pipe_dir)
-          FileUtils.rm_r(@pipe_dir, force: true, secure: true)
-        end
+        Channel.teardown
       end
 
-      def spawn_source(a_class, *args, writer: self.class.dasherize(a_class.name))
-        create_pipe(writer)
+      def spawn_process(a_class, *args, process_name: a_class.name, readers: [], writers: [])
+        writers.each { |writer| create_pipe(writer) }
         fork_it do
-          set_process_name(a_class.name)
-          write_pipe(writer) do
-            a_class.new.run(io, *args)
-          end
+          set_process_name(process_name)
+          append_open_io_if_needed(a_class)
+          a_class.new.run(*args, readers: readers, writers: writers)
         end
       end
 
-      def spawn_process(a_class, *args, readers: [], writers: [])
-        writers.each { |writer| create_pipe(writer) }
-        fork_it do
-          set_process_name(a_class.name)
-          open_readers = readers.map { |reader| read_pipe(reader) }
-          open_writers = writers.map { |writer| write_pipe(writer) }
-          begin
-            a_class.new.run(*args, readers: open_readers, writers: open_writers)
-          ensure
-            open_writers.each { |writer| writer.close unless writer.closed? }
-            open_readers.each { |reader| reader.close unless reader.closed? }
-          end
+      def spawn_processes(a_class, *args, process_name: a_class.name, reader_pattern: nil, writers: [])
+        number = last_number(reader_pattern)
+
+        loop do
+          this_reader = pattern_replace(reader_pattern, number)
+          break unless Channel.exists?(this_reader)
+          spawn_process(a_class,
+                        *args,
+                        process_name: process_name,
+                        readers: [this_reader],
+                        writers: Array(writers))
+          number += 1
         end
       end
 
+      def spawn_source(a_class, *args, writer: a_class.dasherize(a_class.name))
+        spawn_process(a_class, *args, process_name: process_name, readers: [], writers: [writer])
+      end
+
+      def spawn_splitter(process_name: 'Splitter', reader: nil, writer: '>output01.txt', number: 2)
+        start = last_number(writer)
+        finish = start + number - 1
+
+        writers = (start..finish).map { |index| pattern_replace(writer, index) }
+
+        spawn_process(Processes::Splitter,
+                      process_name: process_name,
+                      readers: Array(reader),
+                      writers: writers)
+      end
+
+      def spawn_socket_splitter(process_name: 'SocketSplitter', reader: nil, port: 11000, number: 2)
+        spawn_splitter(process_name: process_name,
+                       reader: reader,
+                       writer: "localhost:#{port}",
+                       number: number)
+      end
+
       def spawn_command(a_shell_command)
         command = a_shell_command.dup
         a_shell_command.scan(/\(\((.*?)\)\)/) do |(pipe_name)|
@@ -61,12 +81,18 @@ module DeadlySerious
 
       private
 
+      def append_open_io_if_needed(a_class)
+        a_class.send(:prepend, OpenIo) unless a_class.include?(OpenIo)
+      end
+
+      def create_pipe(pipe_name)
+        Channel.create_pipe(pipe_name)
+      end
+
       # @!group Process Control
 
       def fork_it
-        @ids << fork do
-          yield
-        end
+        @ids << fork { yield }
       end
 
       def wait_children
@@ -83,34 +109,30 @@ module DeadlySerious
       end
 
       # @!endgroup
-      # @!group Channel Helpers
+      # @!group Minor Helpers
 
-      def channel_for(pipe_name)
-        Channel.new(pipe_name, data_dir: @data_dir, pipe_dir: @pipe_dir)
+      def self.dasherize(a_string)
+        a_string.gsub(/(.)([A-Z])/, '\1-\2').downcase.gsub(/\W+/, '-')
       end
 
-      def create_pipe(pipe_name)
-        channel_for(pipe_name).create
-      end
+      def last_number_pattern(a_string)
+        last_number_pattern = /(\d+)[^\d]*$/.match(a_string)
+        raise %(Writer name "#{writer}" should have a number) if last_number_pattern.nil?
 
-      def read_pipe(pipe_name)
-        channel_for(pipe_name).open_reader
+        last_number_pattern[1]
       end
 
-      def write_pipe(pipe_name)
-        channel = channel_for(pipe_name)
-        return channel.open_writer unless block_given?
-
-        channel.open_writer do |io|
-          yield io
-        end
+      def last_number(a_string)
+        last_number_pattern(a_string).to_i
       end
 
-      # @!endgroup
-      # @!group Minor Helpers
+      def pattern_replace(a_string, number)
+        pattern = last_number_pattern(a_string)
+        pattern_length = pattern.size
+        find_pattern = /#{pattern}([^\d]*)$/
+        replace_pattern = "%0.#{pattern_length}d\\1"
 
-      def self.dasherize(a_string)
-        a_string.gsub(/(.)([A-Z])/, '\1-\2').downcase.gsub(/\W+/, '-')
+        a_string.sub(find_pattern, sprintf(replace_pattern, number))
       end
     end
   end

data/lib/deadly_serious/processes/resilient_splitter.rb

@@ -0,0 +1,34 @@
+module DeadlySerious
+  module Processes
+    class ResilientSplitter
+      def initialize
+        @reallocate = false
+        Signal.trap('USR1') do
+          @reallocate = true
+        end
+      end
+
+      def run(readers: [], writers: [])
+        reader = readers.first
+        outputs = writers.dup
+        current = nil
+        reader.each do |line|
+          begin
+            if @reallocate
+              @reallocate = false
+              outputs = writers.dup
+            end
+            current = outputs.first
+            current << line
+            outputs.rotate!
+          rescue Errno::EPIPE => e
+            puts e.inspect
+            outputs.delete(current)
+            raise e if outputs.empty?
+            redo
+          end
+        end
+      end
+    end
+  end
+end

data/lib/deadly_serious/processes/splitter.rb

@@ -1,12 +1,24 @@
 module DeadlySerious
   module Processes
     class Splitter
+      def initialize
+        Signal.trap('USR1') { @outputs = @writers.dup }
+      end
       def run(readers: [], writers: [])
-        reader = readers.first
-        outputs = writers.dup
-        reader.each do |line|
-          outputs.first << line
-          outputs.rotate!
+        @writers ||= writers
+	reader = readers.first
+        @outputs = @writers.dup
+        begin
+          reader.each do |line|
+            @current = @outputs.first
+            @current << line
+            @outputs.rotate!
+          end
+        rescue => e
+          puts e.inspect
+          @outputs.delete(@current)
+          raise e if @outputs.empty?
+          retry
         end
       end
     end

data/lib/deadly_serious/version.rb

@@ -1,3 +1,3 @@
 module DeadlySerious
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

data/scripts/update_ctags.sh

@@ -0,0 +1 @@
+ctags -R . --exclude=.git

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deadly_serious
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Ronie Uliana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-12 00:00:00.000000000 Z
+date: 2013-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,20 +38,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: yard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -84,13 +70,17 @@ files:
 - lib/deadly_serious/engine/channel.rb
 - lib/deadly_serious/engine/json_io.rb
 - lib/deadly_serious/engine/json_process.rb
+- lib/deadly_serious/engine/lazy_io.rb
+- lib/deadly_serious/engine/open_io.rb
 - lib/deadly_serious/engine/push_pop.rb
 - lib/deadly_serious/engine/push_pop_send.rb
 - lib/deadly_serious/engine/spawner.rb
 - lib/deadly_serious/processes/db_source.rb
 - lib/deadly_serious/processes/joiner.rb
+- lib/deadly_serious/processes/resilient_splitter.rb
 - lib/deadly_serious/processes/splitter.rb
 - lib/deadly_serious/version.rb
+- scripts/update_ctags.sh
 homepage: https://github.com/ruliana/deadly_serious
 licenses:
 - MIT
@@ -119,4 +109,3 @@ summary: Flow Based Programming engine that relies on named pipes and Linux proc
   with the Operating System, i.e., the S.O. is *part* of the program, it's not something
   *below* it.
 test_files: []
-has_rdoc: