pants 0.1.0

data/bin/pants

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+
+require 'thor'
+require_relative '../lib/pants'
+require_relative '../lib/pants/version'
+
+
+class PantsRunner < Thor
+  desc "dup [URI]", "read on the URI and send to --destinations"
+  long_desc <<-LONGDESC
+This will read in any data given by the URI and redirect it to all of the I/O\x5
+objects given by --destinations.  Specify all I/O objects as URIs in the form:\x5
+
+  udp://127.0.0.1:1234, file:///home/bob/BarrellRoll.avi
+
+...although files can be given without using the URI form.
+
+Separate multiple destinations by a space:
+
+  $ pants udp://127.0.0.1:1234 --dest=udp://239.0.0.1:1234 udp://10.0.0.1:1234
+
+  LONGDESC
+  option :dest, type: :array, required: true
+  option :verbose, type: :boolean
+  def dup(read_uri)
+    Pants.log = options[:verbose] ? true : false
+
+    add_writers = lambda do |reader|
+      options[:dest].each do |write_uri|
+        reader.write_to(write_uri)
+      end
+    end
+
+    Pants.read(read_uri, &add_writers)
+  end
+
+  desc "version", "print the version of pants"
+  def version
+    puts <<-INFO
+---------------------------
+|    ||      O     ||     |     # => pants v#{Pants::VERSION}
+---------------------------     # => https://github.com/turboladen/pants
+|           | |           |     # => #{RUBY_DESCRIPTION}
+|           | |           |
+|           |_|           |
+|           / \\           |
+|          |   |          |
+|          |   |          |
+|          |   |          |
+|          |   |          |
+|__________|   |__________|
+
+INFO
+  end
+end
+
+
+PantsRunner.start

data/lib/pants.rb

@@ -0,0 +1,84 @@
+require_relative 'pants/core'
+require_relative 'pants/version'
+Dir[File.dirname(__FILE__) + "/readers/*.rb"].each { |f| require f }
+Dir[File.dirname(__FILE__) + "/writers/*.rb"].each { |f| require f }
+require_relative 'pants/seam'
+
+
+# This base class provides some helpers for doing quick, non-complex reading
+# and writing.  Check docs on readers and writers for more information.
+class Pants
+
+  DEFAULT_URI_TO_READER_MAP = [
+    { uri_scheme: nil, klass: Pants::Readers::FileReader, args: [:path] },
+    { uri_scheme: 'file', klass: Pants::Readers::FileReader, args: [:path] },
+    { uri_scheme: 'udp', klass: Pants::Readers::UDPReader, args: [:host, :port] }
+  ]
+
+  DEFAULT_URI_TO_WRITER_MAP = [
+    { uri_scheme: nil, klass: Pants::Writers::FileWriter, args: [:path] },
+    { uri_scheme: 'file', klass: Pants::Writers::FileWriter, args: [:path] },
+    { uri_scheme: 'udp', klass: Pants::Writers::UDPWriter, args: [:host, :port] }
+  ]
+
+  # The list of mappings of URIs to Reader class types.  These mappings allow
+  # Pants to look up the URI scheme and find what type of object should be
+  # created when creating a Reader by giving it a URI.  It also defines the
+  # arguments that the Reader class takes; these should Symbols that represent
+  # names of methods that can be called on objects of the URI type.
+  #
+  # You can register new mappings here by pushing new mappings to the list.
+  # Mappings should be in the form:
+  #   { uri_scheme: 'my_scheme', klass: MyReaderClass, args: [:arg] }
+  #
+  # Note that if you're wanting to add to this list, and URI doesn't recognize
+  # the URI scheme that you're adding for, you'll need to define that within
+  # URI.  An example is given here: http://www.ruby-doc.org/stdlib-1.9.3/libdoc/uri/rdoc/URI.html
+  #
+  # If you want to use your own reader but don't want to go through all of this
+  # hassle, you can add your reader using a different method.  See the docs for
+  # Pants::Core for more info.
+  #
+  # @return [Array<Hash>] The list of mappings.
+  def self.readers
+    @readers ||= DEFAULT_URI_TO_READER_MAP
+  end
+
+  # The list of mappings of URIs to Writer class types.  See the docs for
+  # .readers for more info.
+  #
+  # @return [Array<Hash>] The list of mappings.
+  # @see Pants.readers
+  def self.writers
+    @writers ||= DEFAULT_URI_TO_WRITER_MAP
+  end
+
+  # Convenience method; doing something like:
+  #
+  #   pants = Pants::Core.new
+  #   reader = pants.read('udp://0.0.0.0:1234')
+  #   reader.add_writer('udp://1.2.3.4:5999')
+  #   reader.add_writer('udp_data.raw')
+  #   pants.run
+  #
+  # ...becomes:
+  #
+  #   Pants.read('udp://0.0.0.1234') do |seam|
+  #     seam.add_writer('udp://1.2.3.4:5999')
+  #     seam.add_writer('udp_data.raw')
+  #   end
+  #
+  # @param [String] uri Takes a URI ('udp://...' or 'file://...') or the path
+  #   to a file.
+  def self.read(uri, &block)
+    pants = Pants::Core.new(&block)
+
+    if uri.kind_of? Pants::Readers::BaseReader
+      pants.add_reader(uri)
+    else
+      pants.read(uri)
+    end
+
+    pants.run
+  end
+end

data/lib/pants/core.rb

@@ -0,0 +1,216 @@
+require 'uri'
+require 'eventmachine'
+require_relative 'error'
+require_relative 'logger'
+
+Dir[File.dirname(__FILE__) + "/readers/*.rb"].each { |f| require f }
+Dir[File.dirname(__FILE__) + "/writers/*.rb"].each { |f| require f }
+require_relative 'seam'
+
+
+class Pants
+
+  # A single Core object is necessary for Pants to run.  Root-level readers are
+  # attached to the core, then writers are attached to those readers.  It's
+  # main job, other than giving a home to readers, is to handle the order of
+  # starting up and shutting of the readers and writers so that no data is lost.
+  class Core
+    include LogSwitch::Mixin
+
+    # @return [Array] The list of readers that are reading data.
+    attr_reader :readers
+
+    def initialize(&block)
+      setup_signals
+      @readers = []
+
+      @convenience_block = block
+    end
+
+    # One method of adding a Reader to the Core.  Use this method to make code
+    # reader nicer when reading something that's expressed as a URI.
+    #
+    # @example
+    #   core = Pants::Core.new
+    #   core.read 'udp://10.2.3.4:9000'
+    #
+    # @param [String,URI] uri The URI to the object to read.  Can be a file:///,
+    #   udp://, or a string with the path to a file.
+    #
+    # @return [Pants::Reader] The newly created reader.
+    def read(uri)
+      begin
+        uri = uri.is_a?(URI) ? uri : URI(uri)
+      rescue URI::InvalidURIError
+        @readers << new_reader_from_uri(nil, callback)
+      else
+        @readers << new_reader_from_uri(uri, callback)
+      end
+
+      @convenience_block.call(@readers.last) if @convenience_block
+
+      @readers.last
+    end
+
+    # One method of adding a Reader to the Core.  Use this method to add an
+    # a) already instantiated Reader object, or b) a Reader from a class of
+    # Reader objects.
+    #
+    # @example Add using class and init variables
+    #   core = Pants::Core.new
+    #   core.add_reader(Pants::Readers::UDPReader, '10.2.3.4', 9000)
+    #
+    # @example Add using an already instantiated Reader object
+    #   core = Pants::Core.new
+    #   reader = Pants::Readers::UDPReader.new('10.2.3.4', 9000, core.callback)
+    #   core.add_reader(reader)
+    #
+    # Notice how using the last method requires you to pass in the core's
+    # callback method--this is probably one reason for avoiding this method of
+    # adding a reader, yet remains available for flexibility.
+    #
+    # @param [Class,Pants::Reader] obj Either the class of a Reader to create,
+    #   or an already created Reader object.
+    # @param [*] args Any arguments that need to be used for creating the
+    #   Reader.
+    def add_reader(obj, *args)
+      if obj.is_a? Class
+        @readers << obj.new(*args, callback)
+      elsif obj.kind_of? Pants::Readers::BaseReader
+        @readers << obj
+      else
+        raise Pants::Error, "Don't know how to add a reader of type #{obj}"
+      end
+
+      @convenience_block.call(@readers.last) if @convenience_block
+
+      @readers.last
+    end
+
+    # Creates an EventMachine::Callback method that other Readers, Writers, and
+    # others can use for letting the Core know when it can shutdown.  Those
+    # Readers, Writers, etc. should handle calling this callback when they're
+    # done doing what they need to do.
+    #
+    # @return [EventMachine::Callback]
+    def callback
+      EM.Callback do
+        if @readers.none?(&:running?)
+          EM.stop_event_loop
+        end
+      end
+    end
+
+    # Starts the EventMachine reactor, the reader and the writers.
+    def run
+      raise Pants::Error, "No readers added yet" if @readers.empty?
+
+      starter = proc do
+        puts "Pants v#{Pants::VERSION}"
+        puts ">> Reading from #{@readers.size} readers"
+
+        @readers.each_with_index do |reader, i|
+          puts ">> reader#{i}:  Starting read on: #{reader.read_object}"
+          puts ">> reader#{i}:  Writing to #{reader.writers.size} writers"
+
+          reader.writers.each_with_index do |writer, j|
+            puts ">> reader#{i}writer#{j}:  #{writer.write_object}"
+          end
+        end
+
+        EM::Iterator.new(@readers).each do |reader, iter|
+          reader.start
+          iter.next
+        end
+      end
+
+      if EM.reactor_running?
+        log "Joining reactor..."
+        starter.call
+      else
+        log "Starting reactor..."
+        EM.run(&starter)
+      end
+    end
+
+    # Tells the reader to signal to its writers that it's time to finish.
+    def stop!
+      puts "Stop called.  Closing readers and writers..."
+
+      if @readers.none?(&:running?)
+        puts "No readers are running; nothing to do."
+      else
+        puts "Stopping readers:"
+
+        @readers.each do |reader|
+          puts "\t#{reader}" if reader.running?
+        end
+
+        @readers.each(&:stop!)
+      end
+    end
+
+    # Stop, then run.
+    def restart
+      stop!
+      puts "Restarting..."
+      run
+    end
+
+    private
+
+    # Register signals:
+    # * TERM & QUIT calls +stop+ to shutdown gracefully.
+    # * INT calls <tt>stop!</tt> to force shutdown.
+    # * HUP calls <tt>restart</tt> to ... surprise, restart!
+    def setup_signals
+      @trapped_count ||= 0
+
+      stopper = proc do
+        @trapped_count += 1
+        stop!
+
+        # Reset count after 5 seconds
+        EM.add_timer(5) { @trapped_count = 0 }
+      end
+
+      trap('INT')  do
+        stopper.call
+        abort "Multiple INT signals trapped; aborting!" if @trapped_count > 1
+      end
+
+      trap('TERM') { stopper.call }
+
+      unless !!RUBY_PLATFORM =~ /mswin|mingw/
+        trap('QUIT') { stop! }
+        trap('HUP')  { restart }
+      end
+    end
+
+    # @param [URI] uri The URI the Reader is mapped to.
+    #
+    # @return [Pants::Reader] An object of the type that's defined by the URI
+    #   scheme.
+    #
+    # @raise [Pants::Error] If no Reader is mapped to +scheme+.
+    def new_reader_from_uri(uri, callback)
+      reader_to_use = if uri.nil?
+        Pants.readers.find { |reader| reader[:uri_scheme].nil? }
+      else
+        Pants.readers.find { |reader| reader[:uri_scheme] == uri.scheme }
+      end
+
+      unless reader_to_use
+        raise ArgumentError, "No reader found with URI scheme: #{uri.scheme}"
+      end
+
+      args = if reader_to_use[:args]
+        reader_to_use[:args].map { |arg| uri.send(arg) }
+      else
+        []
+      end
+
+      reader_to_use[:klass].new(*args, callback)
+    end
+  end
+end

data/lib/pants/error.rb

@@ -0,0 +1,5 @@
+class Pants
+  class Error < StandardError
+
+  end
+end

data/lib/pants/logger.rb

@@ -0,0 +1,8 @@
+require 'log_switch'
+
+
+class Pants
+  extend LogSwitch
+end
+
+Pants.log_class_name = true

data/lib/pants/network_helpers.rb

@@ -0,0 +1,25 @@
+require 'ipaddr'
+require 'socket'
+
+
+class Pants
+
+  # Methods that are shared by Readers and Writers that deal with networking.
+  module NetworkHelpers
+
+    private
+
+    # Sets Socket options to allow for multicasting.
+    #
+    # @param [String] ip The IP address to add to the multicast group.
+    def setup_multicast_socket(ip)
+      set_membership(::IPAddr.new(ip).hton + ::IPAddr.new('0.0.0.0').hton)
+    end
+
+    # @param [String] membership The network byte ordered String that represents
+    #   the IP(s) that should join the membership group.
+    def set_membership(membership)
+      set_sock_opt(Socket::IPPROTO_IP, Socket::IP_ADD_MEMBERSHIP, membership)
+    end
+  end
+end

data/lib/pants/readers/base_reader.rb

@@ -0,0 +1,302 @@
+require_relative '../logger'
+
+
+class Pants
+  module Readers
+    class BaseReader
+      include LogSwitch::Mixin
+
+      # @return [Array] The list of Writers attached to the Reader.
+      attr_reader :writers
+
+      # @return [EventMachine::Channel] The channel that Writers should subscribe
+      #   to.
+      attr_reader :write_to_channel
+
+      # @return [EventMachine::Callback] The callback from Core that should be
+      #   called when the Reader is done reading.
+      attr_reader :core_stopper_callback
+
+      # @param [EventMachine::Callback] core_stopper_callback This gets called when all
+      #   reading is done and the writers have written out all their data.  It
+      #   signals to the caller that the job of the reader is all done.  For
+      #   first level readers (readers that are not Tees), this lets Pants check
+      #   all existing Readers to see if they're done, so it can know when to stop
+      #   the reactor.
+      #
+      def initialize(core_stopper_callback)
+        @writers = []
+        @write_to_channel = EM::Channel.new
+        @core_stopper_callback = core_stopper_callback
+        @read_object ||= nil
+        @starter = nil
+        @stopper = nil
+        @running = false
+      end
+
+      # Starts all of the writers, then starts the reader.  Child readers must
+      # call this to make sure the writers are all running and ready for data
+      # before the reader starts pushing data onto its Channel.
+      #
+      # @param [EventMachine::Callback] callback Once all writers are up and
+      #   running, this gets called, letting the caller know all Writers are up
+      #   and running.  This should contain all code that the child Reader wants
+      #   to execute on start.
+      def start(callback)
+        start_loop = EM.tick_loop do
+          if @writers.empty? || @writers.all?(&:running?)
+            :stop
+          end
+        end
+        start_loop.on_stop { callback.call }
+
+        log "Starting writers for reader #{self.__id__}..."
+        EM::Iterator.new(@writers).each do |writer, iter|
+          writer.start
+          iter.next
+        end
+      end
+
+      # Calls the reader's #stopper, thus forcing the reader to shutdown.  For
+      # readers that intend to read a finite amount of data, the Reader should
+      # call the #stopper when it's done; for readers that read a non-stop stream
+      # (i.e. like an open socket), this gets called by OS signals (i.e. if you
+      # ctrl-c).
+      def stop!
+        stopper.call
+      end
+
+      # Allows for adding "about me" info, depending on the reader type.  This
+      # info is printed out when Pants starts, so you know get confirmation of
+      # what you're about to do.  If you don't define this in your reader, nothing
+      # will be printed out.
+      #
+      # @return [String] A String that identifies what the reader is reading
+      #   from.
+      def read_object
+        if @read_object
+          @read_object
+        else
+          warn "No read_object info has been defined for this reader."
+        end
+      end
+
+      # @return [Boolean]
+      def running?
+        @running
+      end
+
+      # @param [String] uri The URI to the object to read.  Can be of URI type
+      #   that's defined in Pants.writers.
+      #
+      # @return [Pants::Writers::BaseWriter] The newly created writer.
+      def write_to(uri)
+        begin
+          uri = uri.is_a?(URI) ? uri : URI(uri)
+        rescue URI::InvalidURIError
+          @writers << new_writer_from_uri(nil, @write_to_channel)
+        else
+          @writers << new_writer_from_uri(uri, @write_to_channel)
+        end
+
+        @writers.last
+      end
+
+      # One method of adding a Writer to the Reader.  Use this method to add an
+      # a) already instantiated Writer object, or b) a Writers from a class of
+      # Writer objects.
+      #
+      # @example Add using class and init variables
+      #   core = Pants::Core.new
+      #   core.read 'udp://10.2.3.4:9000'
+      #   core.add_writer(Pants::Writers::UDPWriter, '10.5.6.7', 9000)
+      #
+      # @example Add using an already instantiated Writer object
+      #   core = Pants::Core.new
+      #   reader = core.read 'udp://10.2.3.4:9000'
+      #   writer = Pants::Writers::UDPWriter.new('10.5.6.7', 9000, reader.write_to_channel)
+      #   core.add_writer(writer)
+      #
+      # Notice how using the last method requires you to pass in the channel
+      # that the reader is pushing data to--this is probably one reason for
+      # avoiding this method of adding a writer, yet remains available for
+      # flexibility.
+      #
+      # @param [Class,Pants::Reader] obj Either the class of a Writer to create,
+      #   or an already created Writer object.
+      #
+      # @param [*] args Any arguments that need to be used for creating the
+      #   Writer.
+      def add_writer(obj, *args)
+        if obj.is_a? Class
+          @writers << obj.new(*args, @write_to_channel)
+        elsif obj.kind_of? Pants::Writers::BaseWriter
+          @writers << obj
+        else
+          raise Pants::Error, "Don't know how to add a writer of type #{obj}"
+        end
+
+        @writers.last
+      end
+
+      # Removes a writer object from the internal list of writers.
+      #
+      # @example Using URI
+      #   reader.writers    # => [<Pants::Writers::FileWriter @file_path='./testfile'...>]
+      #   reader.remove_writer('./testfile')
+      #   reader.writers    # => []
+      #
+      # @example Using class and args as key/value pairs
+      #   reader.writers    # => [<Pants::Writers::FileWriter @file_path='./testfile'...>]
+      #   reader.remove_writer(Pants::Writers::FileWriter, file_path: './testfile')
+      #   reader.writers    # => []
+      #
+      # @param [Class] obj Class of the writer to remove.
+      #
+      # @param [Hash] key_value_pairs Keys are methods to be called on each
+      #   writer and will be checked to see if the return value from that method
+      #   equals the given value.
+      def remove_writer(obj, key_value_pairs=nil)
+        if obj.is_a? Class
+          @writers.delete_if do |writer|
+            writer.is_a?(obj) &&
+              key_value_pairs.all? { |k, v| writer.send(k) == v }
+          end
+        elsif obj.is_a? String
+          writer = begin
+            uri = obj.is_a?(URI) ? obj : URI(obj)
+          rescue URI::InvalidURIError
+            find_writer_from_uri(nil)
+          else
+            find_writer_from_uri(uri)
+          end
+
+          unless writer
+            raise ArgumentError, "No writer found wth URI scheme: #{uri.scheme}"
+          end
+
+          key_value_pairs = if writer[:args]
+            writer[:args].inject({}) do |result, arg|
+              result[arg] = uri.send(arg)
+
+              result
+            end
+          else
+            {}
+          end
+
+          @writers.delete_if do |w|
+            w.is_a?(writer[:klass]) &&
+              key_value_pairs.all? { |k, v| w.send(k) == v }
+          end
+        end
+      end
+
+      # Allows for adding a Pants::Seam (or child) object to the reader's list
+      # of internal writers.  For more info on Seams, see the docs for
+      # Pants::Seam.
+      #
+      # @param [Pants::Seam] klass The class of the Pants::Seam object to
+      #   create.
+      #
+      # @return [Pants::Seam] The seam that was just created.
+      #
+      # @see Pants::Seam
+      def add_seam(klass, *args)
+        @writers << klass.new(@core_stopper_callback, @write_to_channel, *args)
+
+        @writers.last
+      end
+
+      #---------------------------------------------------------------------------
+      # Protecteds
+      #---------------------------------------------------------------------------
+      protected
+
+      # The block to be called when starting up.  Writers should all have been
+      # added before calling this; if writers are started after this, they won't
+      # get the first bytes that are read (due to start-up time).
+      #
+      # This is used internally by child Readers to signal that they're up and
+      # running.  If implementing your own Reader, make sure to call this.
+      def starter
+        @starter ||= EM.Callback { @running = true }
+      end
+
+      # The callback that gets called when the Reader is done reading.  Tells all
+      # of the associated writers to finish up.
+      #
+      # @return [EventMachine::Callback] The Callback that should get
+      #   called by any Reader when it's done reading.
+      def stopper
+        return @stopper if @stopper
+
+        @stopper = EM.Callback do
+          log "Got called back after finished reading.  Starting shutdown..."
+
+          # remove this next_tick?
+          EM.next_tick do
+            start_loop = EM.tick_loop do
+              if @writers.empty? || @writers.none?(&:running?)
+                :stop
+              end
+            end
+
+            start_loop.on_stop do
+              @running = false
+              puts ">> All done reading on '#{@read_object}'."
+              @core_stopper_callback.call
+            end
+
+            log "Stopping writers for reader #{self.__id__}"
+            EM::Iterator.new(@writers).each do |writer, iter|
+              writer.stop
+              iter.next
+            end
+          end
+        end
+
+        @stopper
+      end
+
+      #---------------------------------------------------------------------------
+      # Privates
+      #---------------------------------------------------------------------------
+      private
+
+      # Creates a Writer based on the mapping defined in Pants.writers.
+      #
+      # @param [URI] uri The URI that defines the Writer.
+      # @param [EventMachine::Channel] read_from_channel The channel that the
+      #   Writer will read from.
+      # @return [Pants::Writer] The newly created Writer.
+      # @raise [ArgumentError] If Pants.writers doesn't contain a mapping for the
+      #   URI to a Writer class.
+      def new_writer_from_uri(uri, read_from_channel)
+        writer_to_use = find_writer_from_uri(uri)
+
+        unless writer_to_use
+          raise ArgumentError, "No writer found wth URI scheme: #{uri.scheme}"
+        end
+
+        args = if writer_to_use[:args]
+          writer_to_use[:args].map { |arg| uri.send(arg) }
+        else
+          []
+        end
+
+        writer_to_use[:klass].new(*args, read_from_channel)
+      end
+
+      # @param [URI] uri The URI that defines the Writer.
+      # @return [Hash] The Hash from Pants.writers that matches the URI.
+      def find_writer_from_uri(uri)
+        if uri.nil?
+          Pants.writers.find { |writer| writer[:uri_scheme].nil? }
+        else
+          Pants.writers.find { |writer| writer[:uri_scheme] == uri.scheme }
+        end
+      end
+    end
+  end
+end