pants 0.1.0

data/lib/pants/readers/file_reader.rb

@@ -0,0 +1,81 @@
+require 'eventmachine'
+require_relative 'base_reader'
+
+
+class Pants
+  module Readers
+    # This is the EventMachine connection that reads the source file and puts
+    # the read data into the data channel so writers can write as they need to.
+    class FileReaderConnection < EventMachine::Connection
+      include LogSwitch::Mixin
+
+      # @param [EventMachine::Channel] write_to_channel The data channel to write
+      #   read data to.
+
+      # @param [EventMachine::Callback] starter Gets called when the
+      #   it's been fulling initialized.
+      #
+      # @param [EventMachine::Callback] stopper Gets called when the
+      #   file-to-read has been fully read.
+      def initialize(write_to_channel, starter, stopper)
+        @write_to_channel = write_to_channel
+        @stopper = stopper
+        @starter = starter
+      end
+
+      def post_init
+        @starter.call
+      end
+
+      # Reads the data and writes it to the data channel.
+      #
+      # @param [String] data The file data to write to the channel.
+      def receive_data(data)
+        log "<< #{data.size}"
+        @write_to_channel << data
+      end
+
+      # Called when the file is done being read.
+      def unbind
+        log "Unbinding, done writing, and notifying the stopper..."
+        @stopper.call
+      end
+    end
+
+
+    # This is the interface for FileReaderConnections.  It controls starting and
+    # stopping the connection.
+    class FileReader < BaseReader
+      include LogSwitch::Mixin
+
+      # @return [String] Path to the file that's being read.
+      attr_reader :file_path
+
+      # @param [String] file_path Path to the file to read.
+      #
+      # @param [EventMachine::Callback] core_stopper_callback The Callback that will get
+      #   called when #stopper is called.  #stopper is called when the whole
+      #   file has been read and pushed to the channel.
+      def initialize(file_path, core_stopper_callback)
+        log "Initializing #{self.class} with file path '#{file_path}'"
+        @read_object = file_path
+        @file_path = file_path
+
+        log "Opening file '#{@file_path}'"
+        @file = File.open(@file_path, 'r')
+
+        super(core_stopper_callback)
+      end
+
+      # Starts reading the file after all writers have been started.
+      def start
+        callback = EM.Callback do
+          log "Adding file '#{@file_path}'..."
+          EM.attach(@file, FileReaderConnection, @write_to_channel, starter, stopper)
+        end
+
+        super(callback)
+      end
+    end
+  end
+end

data/lib/pants/readers/udp_reader.rb

@@ -0,0 +1,80 @@
+require 'eventmachine'
+require_relative 'base_reader'
+require_relative '../network_helpers'
+
+
+class Pants
+  module Readers
+
+    # This is the EventMachine connection that reads on the source IP and UDP
+    # port.  It places all read data onto the data channel.  Allows for unicast or
+    # multicast addresses; it'll detect which to use from the IP you pass in.
+    class UDPReaderConnection < EventMachine::Connection
+      include LogSwitch::Mixin
+      include Pants::NetworkHelpers
+
+      # @param [EventMachine::Channel] write_to_channel The data channel to write
+      #   read data to.
+      #
+      # @param [EventMachine::Callback] starter_callback The callback that
+      #   should get called when the connection has been fully initialized.
+      def initialize(write_to_channel, starter_callback)
+        @write_to_channel = write_to_channel
+        @starter_callback = starter_callback
+        port, ip = Socket.unpack_sockaddr_in(get_sockname)
+
+        if Addrinfo.ip(ip).ipv4_multicast? || Addrinfo.ip(ip).ipv6_multicast?
+          log "Got a multicast address: #{ip}:#{port}"
+          setup_multicast_socket(ip)
+        else
+          log "Got a unicast address: #{ip}:#{port}"
+        end
+      end
+
+      def post_init
+        @starter_callback.call
+      end
+
+      # Reads the data and writes it to the data channel.
+      #
+      # @param [String] data The socket data to write to the channel.
+      def receive_data(data)
+        @write_to_channel << data
+      end
+    end
+
+
+    # This is the interface for UDPReaderConnections.  It controls what happens
+    # when the you want to start it up and stop it.
+    class UDPReader < BaseReader
+      include LogSwitch::Mixin
+
+      # @param [String] host The IP address to read on.
+      #
+      # @param [Fixnum] port The UDP port to read on.
+      #
+      # @param [EventMachine::Callback] core_stopper_callback The Callback that will get
+      #   called when #stopper is called.  Since there is no clear end to when
+      #   to stop reading this I/O, #stopper is never called internally; it must
+      #   be called externally.
+      def initialize(host, port, core_stopper_callback)
+        @read_object = "udp://#{host}:#{port}"
+        @host = host
+        @port = port
+
+        super(core_stopper_callback)
+      end
+
+      # Starts reading on the UDP IP and port and pushing packets to the channel.
+      def start
+        callback = EM.Callback do
+          log "Adding a #{self.class} at #{@host}:#{@port}..."
+          EM.open_datagram_socket(@host, @port, UDPReaderConnection,
+            @write_to_channel, starter)
+        end
+
+        super(callback)
+      end
+    end
+  end
+end

data/lib/pants/seam.rb

@@ -0,0 +1,120 @@
+require_relative 'readers/base_reader'
+
+
+class Pants
+
+  # A Seam is a core Pants object type (like Readers and Writers) that lets you
+  # attach to a Reader, work with the read data, and pass it on to attached
+  # Writers.  It implements buffering by using EventMachine Queues: pop data
+  # off the @read_queue, work with it, then push it onto the @write_queue.  Once
+  # on the @write_queue, the Seam will pass on to all Writers that have been
+  # added to it.
+  #
+  # The @read_queue is wrapped by #read_items, which yields data
+  # chunks from the Reader in, allowing easy access to each bit of data as it
+  # was when it was read in.  The @write_queue is wrapped by #write, which
+  # lets you just give it the data you want to pass on to the attached Writers.
+  #
+  # Seams are particularly useful for working with network data, where if you're
+  # redirecting traffic from one place to another, you may need to alter data
+  # in those packets to make it useful to the receiving ends.
+  class Seam < Pants::Readers::BaseReader
+    include LogSwitch::Mixin
+
+    # @param [EventMachine::Callback] core_stopper_callback The callback that's
+    #   provided by Core.
+    #
+    # @param [EventMachine::Channel] reader_channel The channel from the Reader
+    #   that the Seam is attached to.
+    def initialize(core_stopper_callback, reader_channel)
+      @read_queue = EM::Queue.new
+      @write_queue = EM::Queue.new
+      @write_object ||= nil
+
+      @receives = 0
+      @reads = 0
+      @writes = 0
+      @sends = 0
+
+      reader_channel.subscribe do |data|
+        log "Got data on reader channel"
+        @read_queue << data
+        @receives += data.size
+      end
+
+      super(core_stopper_callback)
+      send_data
+    end
+
+    def start(callback)
+      super(callback)
+
+      starter.call
+    end
+
+    # Make sure you call this (with super()) in your child to ensure read and
+    # write queues are flushed.
+    def stop
+      log "Stopping..."
+      log "receives #{@receives}"
+      log "reads #{@reads}"
+      log "writes #{@writes}"
+      log "sends #{@sends}"
+
+      finish_loop = EM.tick_loop do
+        if @read_queue.empty? && @write_queue.empty?
+          :stop
+        end
+      end
+
+      finish_loop.on_stop { stopper.call }
+    end
+
+    # @return [String] A String that identifies what the writer is writing to.
+    #   This is simply used for displaying info to the user.
+    def write_object
+      if @write_object
+        @write_object
+      else
+        warn "No write_object info has been defined for this writer."
+      end
+    end
+
+    # Call this to read data that was put into the read queue.  It yields one
+    # "item" (however the data was put onto the queue) at a time.  It will
+    # continually yield as there is data that comes in on the queue.
+    #
+    # @param [Proc] block The block to yield items from the reader to.
+    # @yield [item] Gives one item off the read queue.
+    def read_items(&block)
+      processor = proc do |item|
+        block.call(item)
+        @reads += item.size
+        @read_queue.pop(&processor)
+      end
+
+      @read_queue.pop(&processor)
+    end
+
+    # Call this after your Seam child has processed data and is ready to send it
+    # to its writers.
+    #
+    # @param [Object] data
+    def write(data)
+      @write_queue << data
+      @writes += data.size
+    end
+
+    private
+
+    def send_data
+      processor = proc do |data|
+        @write_to_channel << data
+        @sends += data.size
+        @write_queue.pop(&processor)
+      end
+
+      @write_queue.pop(&processor)
+    end
+  end
+end

data/lib/pants/version.rb

@@ -0,0 +1,3 @@
+class Pants
+  VERSION = '0.1.0'
+end

data/lib/pants/writers/base_writer.rb

@@ -0,0 +1,73 @@
+require 'eventmachine'
+require_relative '../logger'
+
+
+class Pants
+  module Writers
+
+    # Provides conventions for creating your own writer that can stop and start
+    # safely.
+    #
+    # You should also consider adding attr_readers/methods for attributes that
+    # are differentiators from other writers of the same type.  This will allow
+    # readers to more easily remove your writer from them.
+    class BaseWriter
+
+      # @param [EventMachine::Channel] read_from_channel The channel that this
+      #   writer should read from.
+      def initialize(read_from_channel)
+        @running = false
+        @read_from_channel = read_from_channel
+        @write_object ||= nil
+        @starter = nil
+        @stopper = nil
+      end
+
+      # This method must be redefined in a child class.  The reader that this
+      # writer is tied to will call this before it starts reading.
+      def start
+        warn "You haven't defined a start method--are you sure this writer does something?"
+      end
+
+      # This method must be redefined in a child class.  The reader that this
+      # writer is tied to will call this when it's done reading whatever it's
+      # reading.
+      def stop
+        warn "You haven't defined a stop method--are you sure you're cleaning up?"
+      end
+
+      # @return [String] A String that identifies what the writer is writing to.
+      #   This is simply used for displaying info to the user.
+      def write_object
+        if @write_object
+          @write_object
+        else
+          warn "No write_object info has been defined for this writer."
+        end
+      end
+
+      # This should get called with #call after the writer is sure to be up
+      # and running, ready for accepting data.
+      #
+      # @return [EventMachine::Callback] The Callback that should get
+      #   called.
+      def starter
+        @starter ||= EM.Callback { @running = true }
+      end
+
+      # This should get called with #call after the writer is done writing
+      # out the data in its channel.
+      #
+      # @return [EventMachine::Callback] The Callback that should get
+      #   called.
+      def stopper
+        @stopper ||= EM.Callback { @running = false }
+      end
+
+      # @return [Boolean] Is the Writer writing data?
+      def running?
+        @running
+      end
+    end
+  end
+end

data/lib/pants/writers/file_writer.rb

@@ -0,0 +1,59 @@
+require_relative 'base_writer'
+
+
+class Pants
+  module Writers
+
+    # This is the interface for FileWriterConnections.  It controls starting,
+    # stopping, and threading the connection.
+    class FileWriter < BaseWriter
+      include LogSwitch::Mixin
+
+      # @return [String] The path to the file that's being written to.
+      attr_reader :file_path
+
+      # @param [EventMachine::Channel] read_from_channel The channel to read data
+      #   from and thus write to file.
+      #
+      # @param [String] file_path The path to write to.
+      def initialize(file_path, read_from_channel)
+        @file = file_path.is_a?(File) ? file_path : File.open(file_path, 'w')
+        @file_path = file_path
+        @write_object = @file_path
+
+        super(read_from_channel)
+      end
+
+      def stop
+        log "Finishing ID #{__id__} and closing file #{@file}"
+        @file.close unless @file.closed?
+        stopper.call
+      end
+
+      def start
+        log "#{__id__} Adding a #{self.class} to write to #{@file_path}"
+
+        EM.defer do
+          @read_from_channel.subscribe do |data|
+            begin
+              bytes_written = @file.write_nonblock(data)
+              log "Wrote normal, #{bytes_written} bytes"
+            rescue IOError
+              log "Finishing writing; only wrote #{bytes_written}"
+
+              unless bytes_written == data.size
+                File.open(@file, 'a') do |file|
+                  file.write_nonblock(data)
+                end
+              end
+            end
+          end
+
+          start_loop = EM.tick_loop { :stop unless @file.closed? }
+          start_loop.on_stop { starter.call }
+        end
+      end
+    end
+  end
+end
+

data/lib/pants/writers/udp_writer.rb

@@ -0,0 +1,125 @@
+require 'socket'
+require_relative 'base_writer'
+require_relative '../network_helpers'
+
+
+class Pants
+  module Writers
+
+    # This is the EventMachine connection that connects the data from the data
+    # channel (put there by the reader you're using) to the IP and UDP port you
+    # want to send it to.
+    class UDPWriterConnection < EM::Connection
+      include LogSwitch::Mixin
+      include Pants::NetworkHelpers
+
+      # Packets get split up before writing if they're over this size.
+      PACKET_SPLIT_THRESHOLD = 1400
+
+      # Packets get split up to this size before writing.
+      PACKET_SPLIT_SIZE = 1300
+
+      # @param [EventMachine::Channel] read_from_channel The channel to expect
+      #   data on and write to the socket.
+      #
+      # @param [String] dest_ip The IP address to send data to.  Can be unicast
+      #   or multicast.
+      #
+      # @param [Fixnum] dest_port The UDP port to send data to.
+      def initialize(read_from_channel, dest_ip, dest_port)
+        @read_from_channel = read_from_channel
+        @dest_ip = dest_ip
+        @dest_port = dest_port
+
+        if Addrinfo.ip(@dest_ip).ipv4_multicast? || Addrinfo.ip(@dest_ip).ipv6_multicast?
+          log "Got a multicast address: #{@dest_ip}:#{@dest_port}"
+          setup_multicast_socket(@dest_ip)
+        else
+          log "Got a unicast address: #{@dest_ip}:#{@dest_port}"
+        end
+      end
+
+      # Sends data received on the data channel to the destination IP and port.
+      # Since data may have been put in to the channel by a File reader (and will
+      # therefore be larger chunks of data than you'll want to send in a packet
+      # over the wire), it will split packets into +PACKET_SPLIT_SIZE+ sized
+      # packets before sending.
+      def post_init
+        @read_from_channel.subscribe do |data|
+          if data.size > PACKET_SPLIT_THRESHOLD
+            log "#{__id__} Got big data: #{data.size}.  Splitting..."
+            io = StringIO.new(data)
+            io.binmode
+
+            begin
+              log "#{__id__} Spliced #{PACKET_SPLIT_SIZE} bytes to socket packet"
+
+              while true
+                new_packet = io.read_nonblock(PACKET_SPLIT_SIZE)
+                send_datagram(new_packet, @dest_ip, @dest_port)
+                new_packet = nil
+              end
+            rescue EOFError
+              send_datagram(new_packet, @dest_ip, @dest_port) if new_packet
+              io.close
+            end
+          else
+            log "Sending data to #{@dest_ip}:#{@dest_port}"
+            send_datagram(data, @dest_ip, @dest_port)
+          end
+        end
+      end
+
+      def receive_data(data)
+        log "Got data (should I?): #{data.size}, port #{@dest_port}, peer: #{get_peername}"
+      end
+    end
+
+
+    # This is the interface to UDPWriterConnections.  It defines what happens
+    # when you want to start it up and stop it.
+    class UDPWriter < BaseWriter
+      include LogSwitch::Mixin
+
+      # @return [String] The IP address that's being written to.
+      attr_reader :host
+
+      # @return [Fixnum] The port that's being written to.
+      attr_reader :port
+
+      # @param [String] host
+      #
+      # @param [Fixnum] port
+      #
+      # @param [EventMachine::Channel] read_from_channel
+      def initialize(host, port, read_from_channel)
+        @host = host
+        @port = port
+        @connection = nil
+        @write_object = "udp://#{@host}:#{@port}"
+
+        super(read_from_channel)
+      end
+
+      # Readies the writer for data to write and waits for data to write.
+      def start
+        log "#{__id__} Adding a #{self.class} at #{@host}:#{@port}..."
+
+        EM.defer do
+          @connection = EM.open_datagram_socket('0.0.0.0', 0, UDPWriterConnection,
+            @read_from_channel, @host, @port)
+
+          start_loop = EM.tick_loop { :stop if @connection }
+          start_loop.on_stop { starter.call }
+        end
+      end
+
+      # Closes the connection and notifies the reader that it's done.
+      def stop
+        log "Finishing ID #{__id__}"
+        @connection.close_connection_after_writing
+        stopper.call
+      end
+    end
+  end
+end