carnivore-http 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 254e80996f94e8fb440fd9e685205764dc4c1d6d
-  data.tar.gz: 56d1e0aab6e50decb27265752c90cb7490daf033
+  metadata.gz: 7f9f4a7936dcc06d41a919c221612c146426790a
+  data.tar.gz: 6153beb3e86f2dd7a3bf7eadad967a3fde8826b5
 SHA512:
-  metadata.gz: 92ef01935a6852996c57cb73b290d67d9ecae93f0b4cff4b96cbc98e6fc25ef2fc31e57ba4c168eb89889055a2615784ef7a4bbfa909f226d7b24e1364593cce
-  data.tar.gz: 0e1e6bea0f6e36e5d305c720aaf83add3c420c61fd6bd6b3b460835096f7b4bf211160ff6a380e76f54f3b8bdea00a120f5b4f8a2696b76293dd1030981f456b
+  metadata.gz: 459ee7c23877b7638eadf34b397dcfacec31237b7b6543ed0aaf7567ac26d155a87a422a1ea309bc7f09aaff0f82702321d51ab25a08e1e8c129f9042172697a
+  data.tar.gz: 029e4ebfc459957bb46db5fc1f3590efa1126d3dedb0e4f2831b70cb480860d4a33d4675d5a87acdd798f00451d3276a0a953c8e181f7ffdbc03072fa3480f7a

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# V0.2.0
+* Add support for message re-delivery and local persistence
+* Add support for authorization
+* Add HTTPS support
+* Add `:http_path` source
+
 # v0.1.8
 * Add better message body handling
 * Update DSL inclusion on subclasses

data/Gemfile.lock

@@ -1,40 +1,60 @@
 PATH
   remote: .
   specs:
-    carnivore-http (0.1.5)
+    carnivore-http (0.1.9)
       blockenspiel
       carnivore (>= 0.1.8)
+      htauth
       reel (~> 0.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    attribute_struct (0.2.8)
     blockenspiel (0.4.5)
-    carnivore (0.2.2)
+    bogo (0.1.6)
+      hashie
+      multi_json
+    bogo-config (0.1.6)
+      attribute_struct
+      bogo (>= 0.1.4, < 1.0)
+      multi_json
+      multi_xml
+    carnivore (0.3.4)
+      bogo-config
       celluloid
       hashie
-      mixlib-config
       multi_json
-    celluloid (0.15.2)
-      timers (~> 1.1.0)
-    celluloid-io (0.15.0)
-      celluloid (>= 0.15.0)
-      nio4r (>= 0.5.0)
-    hashie (2.1.1)
-    http (0.6.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    celluloid-io (0.16.2)
+      celluloid (>= 0.16.0)
+      nio4r (>= 1.1.0)
+    form_data (0.1.0)
+    hashie (3.3.2)
+    highline (1.6.21)
+    hitimes (1.2.2)
+    hitimes (1.2.2-java)
+    htauth (1.1.0)
+      highline (~> 1.6)
+    http (0.7.1)
+      form_data (~> 0.1.0)
       http_parser.rb (~> 0.6.0)
     http_parser.rb (0.6.0)
-    mixlib-config (2.1.0)
-    multi_json (1.10.0)
-    nio4r (1.0.0)
+    http_parser.rb (0.6.0-java)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    nio4r (1.1.0)
+    nio4r (1.1.0-java)
     reel (0.5.0)
       celluloid (>= 0.15.1)
       celluloid-io (>= 0.15.0)
       http (>= 0.6.0.pre)
       http_parser.rb (>= 0.6.0)
       websocket_parser (>= 0.1.6)
-    timers (1.1.0)
-    websocket_parser (0.1.6)
+    timers (4.0.1)
+      hitimes
+    websocket_parser (1.0.0)
 
 PLATFORMS
   java

data/README.md

@@ -6,30 +6,77 @@ Provides HTTP `Carnivore::Source`
 
 ## HTTP
 
+All requests are processed via single source
+
 ```ruby
 require 'carnivore'
 require 'carnivore-http'
 
 Carnivore.configure do
   source = Carnivore::Source.build(
-    :type => :http, :args => {:port => 8080}
+    :type => :http,
+    :args => {
+      :port => 8080
+    }
   )
 end
 ```
 
 ## HTTP with configured end points
 
+All point builder definitions are hooked into source:
+
 ```ruby
 require 'carnivore'
 require 'carnivore-http'
 
 Carnivore.configure do
   source = Carnivore::Source.build(
-    :type => :http_endpoints, :args => {:auto_respond => false}
+    :type => :http_endpoints,
+    :args => {
+      :auto_respond => false
+    }
   )
 end.start!
 ```
 
+## HTTP paths
+
+Multiple sources share same listener, and incoming messages
+are matched based on HTTP method + path
+
+```ruby
+require 'carnivore'
+require 'carnivore-http'
+
+Carnivore.configure do
+  source = Carnivore::Source.build(
+    :type => :http_paths,
+    :args => {
+      :port => 8080,
+      :path => '/test',
+      :method => 'get'
+    }
+  )
+end
+```
+
+## Available options for `:args`
+
+* `:bind` address to bind
+* `:port` port to listen
+* `:auto_respond` confirm request immediately
+* `:ssl` ssl configuration
+  * `:cert` path to cert file
+  * `:key` path to key file
+* `:authorization` access restrictors
+  * `:allowed_origins` list of IP or IP ranges
+  * `:htpasswd` htpasswd for authentication
+  * `:credentials` username/password key pair for authentication
+  * `:valid_on` 'any' match any restrictor, 'all' match all restrictors
+* `:endpoint` specific uri to transmit (can include auth + path)
+* `:method` HTTP method for transmission
+
 # Info
 * Carnivore: https://github.com/carnivore-rb/carnivore
 * Repository: https://github.com/carnivore-rb/carnivore-http

data/carnivore-http.gemspec

@@ -13,5 +13,6 @@ Gem::Specification.new do |s|
   s.add_dependency 'carnivore', '>= 0.1.8'
   s.add_dependency 'reel', '~> 0.5.0'
   s.add_dependency 'blockenspiel'
+  s.add_dependency 'htauth'
   s.files = Dir['**/*']
 end

data/lib/carnivore-http/http.rb

@@ -1,143 +1,37 @@
-require 'reel'
-require 'tempfile'
-require 'carnivore/source'
-require 'carnivore-http/utils'
+require 'carnivore-http'
 
 module Carnivore
   class Source
-
-    # Carnivore HTTP source
-    class Http < Source
-
-      include Carnivore::Http::Utils::Params
-
-      # @return [Hash] source arguments
-      attr_reader :args
-
-      # Setup the source
-      #
-      # @params args [Hash] setup arguments
-      def setup(args={})
-        @args = default_args(args)
-      end
-
-      # Default configuration arguments. If hash is provided, it
-      # will be merged into the default arguments.
-      #
-      # @param args [Hash]
-      # @return [Hash]
-      def default_args(args={})
-        Smash.new(
-          :bind => '0.0.0.0',
-          :port => '3000',
-          :auto_respond => true
-        ).merge(args)
-      end
-
-      # Tranmit message. The transmission can be a response
-      # back to an open connection, or a request to a remote
-      # source (remote carnivore-http source generally)
-      #
-      # @param message [Object] message to transmit
-      # @param extras [Object] argument list
-      def transmit(message, *extra)
-        options = extra.detect{|x| x.is_a?(Hash)} || {}
-        orig = extra.detect{|x| x.is_a?(Carnivore::Message)}
-        con = options[:connection]
-        if(orig && con.nil?)
-          con = orig[:message][:connection]
-        end
-        if(con) # response
-          payload = message.is_a?(String) ? message : MultiJson.dump(message)
-          # TODO: add `options` options for marshaling: json/xml/etc
-          debug "Transmit response type with payload: #{payload}"
-          con.respond(options[:code] || :ok, payload)
-        else # request
-          url = File.join("http://#{args[:bind]}:#{args[:port]}", options[:path].to_s)
-          method = (options[:method] || :post).to_sym
-          if(options[:headers])
-            base = HTTP.with_headers(options[:headers])
-          else
-            base = HTTP
-          end
-          payload = message.is_a?(String) ? message : MultiJson.dump(message)
-          debug "Transmit request type with payload: #{payload}"
-          base.send(method, url, :body => payload)
-        end
-      end
-
-      # Confirm processing of message
-      #
-      # @param message [Carnivore::Message]
-      # @param args [Hash]
-      # @option args [Symbol] :code return code
-      def confirm(message, args={})
-        code = args.delete(:code) || :ok
-        args[:response_body] = 'Thanks' if code == :ok && args.empty?
-        debug "Confirming #{message} with: Code: #{code.inspect} Args: #{args.inspect}"
-        message[:message][:request].respond(code, args[:response_body] || args)
-      end
+    class Http < HttpSource
 
       # Process requests
       def process(*process_args)
-        srv = Reel::Server::HTTP.supervise(args[:bind], args[:port]) do |con|
-          con.each_request do |req|
-            begin
-              msg = build_message(con, req)
-              callbacks.each do |name|
-                c_name = callback_name(name)
-                debug "Dispatching #{msg} to callback<#{name} (#{c_name})>"
-                callback_supervisor[c_name].call(msg)
+        unless(@processing)
+          @processing = true
+          srv = build_listener do |con|
+            con.each_request do |req|
+              begin
+                msg = build_message(con, req)
+                msg = format(msg)
+                if(authorized?(msg))
+                  callbacks.each do |name|
+                    c_name = callback_name(name)
+                    debug "Dispatching #{msg} to callback<#{name} (#{c_name})>"
+                    callback_supervisor[c_name].call(msg)
+                  end
+                  req.respond(:ok, 'So long, and thanks for all the fish!') if args[:auto_respond]
+                else
+                  req.respond(:unauthorized, 'You are not authorized to perform requested action!')
+                end
+              rescue => e
+                req.respond(:bad_request, "Failed to process request -> #{e}")
               end
-              req.respond(:ok, 'So long, and thanks for all the fish!') if args[:auto_respond]
-            rescue => e
-              req.respond(:bad_request, "Failed to process request -> #{e}")
             end
           end
-        end
-      end
-
-      # Size limit for inline body
-      BODY_TO_FILE_SIZE = 1024 * 10 # 10K
-
-      # Build message hash from request
-      #
-      # @param con [Reel::Connection]
-      # @param req [Reel::Request]
-      # @return [Hash]
-      # @note
-      #   if body size is greater than BODY_TO_FILE_SIZE
-      #   the body will be a temp file instead of a string
-      def build_message(con, req)
-        msg = Smash.new(
-          :request => req,
-          :headers => Smash[
-            req.headers.map{ |k,v| [k.downcase.tr('-', '_'), v]}
-          ],
-          :connection => con,
-          :query => parse_query_string(req.query_string)
-        )
-        if(msg[:headers][:content_type] == 'application/json')
-          msg[:body] = MultiJson.load(
-            req.body.to_s
-          )
-        elsif(msg[:headers][:content_type] == 'application/x-www-form-urlencoded')
-          msg[:body] = parse_query_string(
-            req.body.to_s
-          )
-          if(msg[:body].size == 1 && msg[:body].values.first.is_a?(Array) && msg[:body].values.first.empty?)
-            msg[:body] = msg[:body].keys.first
-          end
-        elsif(msg[:headers][:content_length].to_i > BODY_TO_FILE_SIZE)
-          msg[:body] = Tempfile.new('carnivore-http')
-          while((chunk = req.body.readpartial(2048)))
-            msg[:body] << chunk
-          end
-          msg[:body].rewind
+          true
         else
-          msg[:body] = req.body.to_s
+          false
         end
-        format(msg)
       end
 
     end

data/lib/carnivore-http/http_endpoints.rb

@@ -76,28 +76,39 @@ module Carnivore
         set_points
       end
 
-      # Start processing
-      def connect
-        info 'Override initialization process startup. Force enabling processing.'
-        async.process
+      # Always auto start
+      def auto_process?
+        true
       end
 
       # Process requests
       def process(*process_args)
-        srv = Reel::Server::HTTP.supervise(args[:bind], args[:port]) do |con|
-          con.each_request do |req|
-            begin
-              msg = build_message(con, req)
-              unless(@points.deliver(msg))
-                warn "No match found for request: #{msg}"
-                req.respond(:ok, 'So long, and thanks for all the fish!')
+        unless(processing)
+          @processing = true
+          srv = build_listener do |con|
+            con.each_request do |req|
+              begin
+                msg = build_message(con, req)
+                msg = format(msg)
+                if(authorized?(msg))
+                  unless(@points.deliver(msg))
+                    warn "No match found for request: #{msg} (path: #{msg[:message][:request].url})"
+                    debug "Unmatched message (#{msg}): #{msg.inspect}"
+                    req.respond(:not_found, 'So long, and thanks for all the fish!')
+                  end
+                else
+                  req.respond(:unauthorized, 'You are not authorized to perform requested action!')
+                end
+              rescue => e
+                error "Failed to process message: #{e.class} - #{e}"
+                debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+                req.respond(:bad_request, 'Failed to process request')
               end
-            rescue => e
-              error "Failed to process message: #{e.class} - #{e}"
-              debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
-              req.respond(:bad_request, 'Failed to process request')
             end
           end
+          true
+        else
+          false
         end
       end
 

data/lib/carnivore-http/http_paths.rb

@@ -0,0 +1,99 @@
+require 'bogo'
+require 'carnivore-http/http'
+
+module Carnivore
+  class Source
+
+    # Carnivore HTTP paths
+    class HttpPaths < HttpSource
+
+      finalizer :halt_listener
+      include Bogo::Memoization
+
+      # @return [String] end point path
+      attr_reader :http_path
+      # @return [Symbol] http method
+      attr_reader :http_method
+
+      # Kill listener on shutdown
+      def halt_listener
+        listener = memoize("#{args[:bind]}-#{args[:port]}", :global){ nil }
+        if(listener && listener.alive?)
+          listener.terminate
+        end
+        unmemoize("#{args[:bind]}-#{args[:port]}", :global)
+        unmemoize("#{args[:bind]}-#{args[:port]}-queues", :global)
+      end
+
+      # Setup message queue for source
+      def setup(*_)
+        @http_path = args.fetch(:path, '/')
+        @http_method = args.fetch(:method, 'get').to_s.downcase.to_sym
+        if(message_queues[queue_key])
+          raise ArgumentError.new "Conflicting HTTP path source provided! path: #{http_path} method: #{http_method}"
+        else
+          message_queues[queue_key] = Queue.new
+        end
+        super
+      end
+
+      # Setup the HTTP listener source
+      def connect
+        start_listener!
+      end
+
+      # @return [Queue] Message queue
+      def message_queues
+        memoize("#{args[:bind]}-#{args[:port]}-queues", :global) do
+          Smash.new
+        end
+      end
+
+      # @return [String]
+      def queue_key
+        "#{http_path}-#{http_method}"
+      end
+
+      # @return [Queue]
+      def message_queue
+        message_queues[queue_key]
+      end
+
+      # Start the HTTP(S) listener
+      def start_listener!
+        memoize("#{args[:bind]}-#{args[:port]}", :global) do
+          build_listener do |con|
+            con.each_request do |req|
+              begin
+                msg = build_message(con, req)
+                msg_queue = message_queues["#{req.path}-#{req.method.to_s.downcase}"]
+                if(msg_queue)
+                  if(authorized?(msg))
+                    msg_queue << msg
+                    req.respond(:ok, 'So long and thanks for all the fish!')
+                  else
+                    req.respond(:unauthorized, 'You are not authorized to perform requested action!')
+                  end
+                else
+                  req.respond(:not_found, 'Requested path not found!')
+                end
+              rescue => e
+                req.respond(:bad_request, "Failed to process request -> #{e}")
+              end
+            end
+          end
+        end
+      end
+
+      # @return [Object]
+      def receive(*_)
+        val = nil
+        until(val)
+          val = Celluloid::Future.new{ message_queue.pop }.value
+        end
+        val
+      end
+
+    end
+  end
+end

data/lib/carnivore-http/http_source.rb

@@ -0,0 +1,357 @@
+require 'reel'
+require 'tempfile'
+require 'carnivore/source'
+require 'carnivore-http/utils'
+
+module Carnivore
+  class Source
+
+    # Carnivore HTTP source
+    class HttpSource < Source
+
+      trap_exit :retry_delivery_failure
+
+      include Carnivore::Http::Utils::Params
+
+      # @return [Hash] source arguments
+      attr_reader :args
+      # @return [Carnivore::Http::RetryDelivery]
+      attr_reader :retry_delivery
+      # @return [Array<IPAddr>] allowed request origin addresses
+      attr_reader :auth_allowed_origins
+      # @return [HTAuth::PasswdFile]
+      attr_reader :auth_htpasswd
+
+      # Setup the source
+      #
+      # @params args [Hash] setup arguments
+      def setup(args={})
+        require 'fileutils'
+        @args = default_args(args)
+        @retry_delivery = Carnivore::Http::RetryDelivery.new(retry_directory)
+        self.link retry_delivery
+        if(args.get(:authorization, :allowed_origins))
+          require 'ipaddr'
+          @allowed_origins = [args.get(:authorization, :allowed_origins)].flatten.compact.map do |origin_check|
+            IPAddr.new(origin_check)
+          end
+        end
+        if(args.get(:authorization, :htpasswd))
+          require 'htauth'
+          @auth_htpasswd = HTAuth::PasswdFile.open(
+            args.get(:authorization, :htpasswd)
+          )
+        end
+      end
+
+      # Handle failed retry deliveries
+      #
+      # @param actor [Object] terminated actor
+      # @param reason [Exception] reason for termination
+      # @return [NilClass]
+      def retry_delivery_failure(actor, reason)
+        if(actor == retry_delivery)
+          if(reason)
+            error "Failed RetryDelivery encountered: #{reason}. Rebuilding."
+            @retry_delivery = Carnivore::Http::RetryDelivery.new(retry_directory)
+          else
+            info 'Encountered RetryDelivery failure. No reason so assuming teardown.'
+          end
+        else
+          error "Unknown actor failure encountered: #{reason}"
+        end
+        nil
+      end
+
+      # @return [String, NilClass] directory storing failed messages
+      def retry_directory
+        if(args[:retry_directory])
+          FileUtils.mkdir_p(File.join(args[:retry_directory], name.to_s)).first
+        end
+      end
+
+      # @return [String, NilClass] cache directory for initial writes
+      def retry_write_directory
+        base = retry_directory
+        if(base)
+          FileUtils.mkdir_p(File.join(base, '.write')).first
+        end
+      end
+
+      # Default configuration arguments. If hash is provided, it
+      # will be merged into the default arguments.
+      #
+      # @param args [Hash]
+      # @return [Hash]
+      def default_args(args={})
+        Smash.new(
+          :bind => '0.0.0.0',
+          :port => '3000',
+          :auto_respond => true,
+          :retry_directory => '/tmp/.carnivore-resend'
+        ).merge(args)
+      end
+
+      # Always auto start
+      def auto_process?
+        true
+      end
+
+      # Message is authorized for processing
+      #
+      # @param message [Carnivore::Message]
+      # @return [TrueClass, FalseClass]
+      # @note Authorization is driven via the source configuration.
+      #   Valid structure looks like:
+      #     {
+      #       :type => 'http',
+      #       :args => {
+      #         :authorization => {
+      #           :allowed_origins => ['127.0.0.1', '192.168.0.2', '192.168.6.0/24'],
+      #           :htpasswd => '/path/to/htpasswd.file',
+      #           :credentials => {
+      #             :username1 => 'password1'
+      #           },
+      #           :valid_on => :all # or :any
+      #         }
+      #       }
+      #     }
+      #   When multiple authorization items are provided, the
+      #   `:valid_on` will define behavior. It will default to `:all`.
+      def authorized?(message)
+        if(args.fetch(:authorization))
+          valid_on = args.fetch(:authorization, :valid_on, :all).to_sym
+          case valid_on
+          when :all
+            allowed_origin?(message) &&
+              allowed_htpasswd?(message) &&
+              allowed_credentials?(message)
+          when :any
+            allowed_origin?(message) ||
+              allowed_htpasswd?(message) ||
+              allowed_credentials?(message)
+          when :none
+            true
+          else
+            raise ArgumentError.new "Unknown authorization `:valid_on` provided! Given: #{valid_on}. Allowed: `any` or `all`"
+          end
+        else
+          true
+        end
+      end
+
+      # Check if message is allowed based on htpasswd file
+      #
+      # @param message [Carnivore::Message]
+      # @return [TrueClass, FalseClass]
+      def allowed_htpasswd?(message)
+        if(auth_htpasswd)
+          entry = auth_htpasswd.fetch(message[:message][:authentication][:username])
+          if(entry)
+            entry.authenticated?(message[:message][:authentication][:password])
+          else
+            false
+          end
+        else
+          true
+        end
+      end
+
+      # Check if message is allowed based on config credentials
+      #
+      # @param message [Carnivore::Message]
+      # @return [TrueClass, FalseClass]
+      def allowed_credentials?(message)
+        if(creds = args.get(:authorization, :credentials))
+          creds[message[:message][:authentication][:username]] == message[:message][:authentication][:password]
+        else
+          true
+        end
+      end
+
+      # Check if message is allowed based on origin
+      #
+      # @param message [Carnivore::Message]
+      # @return [TrueClass, FalseClass]
+      def allowed_origin?(message)
+        if(auth_allowed_origins)
+          !!auth_allowed_origins.detect do |allowed_check|
+            allowed_check.include?(message[:message][:origin])
+          end
+        else
+          true
+        end
+      end
+
+      # Tranmit message. The transmission can be a response
+      # back to an open connection, or a request to a remote
+      # source (remote carnivore-http source generally)
+      #
+      # @param message [Object] message to transmit
+      # @param extras [Object] argument list
+      def transmit(message, *extra)
+        options = extra.detect{|x| x.is_a?(Hash)} || {}
+        orig = extra.detect{|x| x.is_a?(Carnivore::Message)}
+        con = options[:connection]
+        if(orig && con.nil?)
+          con = orig[:message][:connection]
+        end
+        if(con) # response
+          payload = message.is_a?(String) ? message : MultiJson.dump(message)
+          # TODO: add `options` options for marshaling: json/xml/etc
+          code = options.fetch(:code, :ok)
+          info "Transmit response type with code: #{code}"
+          con.respond(code, payload)
+        else # request
+          if(args[:endpoint])
+            url = args[:endpoint]
+          else
+            url = "http#{'s' if args[:ssl]}://#{args[:bind]}"
+            if(args[:port])
+              url << ":#{args[:port]}"
+            end
+            url = URI.join(url, args.fetch(:path, '/')).to_s
+          end
+          if(options[:path])
+            url = URI.join(url, options[:path].to_s)
+          end
+          method = options.fetch(:method,
+            args.fetch(:method, :post)
+          ).to_s.downcase.to_sym
+          message_id = message.is_a?(Hash) ? message.fetch(:id, Celluloid.uuid) : Celluloid.uuid
+          payload = message.is_a?(String) ? message : MultiJson.dump(message)
+          info "Transmit request type for Message ID: #{message_id}"
+          async.perform_transmission(message_id.to_s, payload, method, url, options.fetch(:headers, {}))
+        end
+      end
+
+      # Transmit message to HTTP endpoint
+      #
+      # @param message_id [String]
+      # @param payload [String] serialized payload
+      # @param method [Symbol] HTTP method (:get, :post, etc)
+      # @param url [String] endpoint URL
+      # @param headers [Hash] request headers
+      # @return [NilClass]
+      def perform_transmission(message_id, payload, method, url, headers={})
+        write_for_retry(message_id, payload, method, url, headers)
+        retry_delivery.async.attempt_redelivery(message_id)
+        nil
+      end
+
+      # Persist message if enabled for send retry
+      #
+      # @param message_id [String] ID of originating message
+      # @param payload [String] serialized payload
+      # @param method [Symbol] HTTP method (:get, :post, etc)
+      # @param url [String] endpoint URL
+      # @param headers [Hash] request headers
+      # @return [TrueClass, FalseClass] message persisted
+      def write_for_retry(message_id, payload, method, url, headers)
+        data = {
+          :message_id => message_id,
+          :payload => payload,
+          :method => method,
+          :url => url,
+          :headers => headers
+        }
+        if(retry_directory)
+          stage_path = File.join(retry_write_directory, "#{message_id}.json")
+          final_path = File.join(retry_directory, File.basename(stage_path))
+          File.open(stage_path, 'w+') do |file|
+            file.write MultiJson.dump(data)
+          end
+          FileUtils.move(stage_path, final_path)
+          info "Failed message (ID: #{message_id}) persisted for resend"
+          true
+        end
+      end
+
+      # Confirm processing of message
+      #
+      # @param message [Carnivore::Message]
+      # @param args [Hash]
+      # @option args [Symbol] :code return code
+      def confirm(message, args={})
+        code = args.delete(:code) || :ok
+        args[:response_body] = 'Thanks' if code == :ok && args.empty?
+        debug "Confirming #{message} with: Code: #{code.inspect} Args: #{args.inspect}"
+        message[:message][:request].respond(code, args[:response_body] || args)
+      end
+
+      # Initialize http listener correctly based on configuration
+      #
+      # @param block [Proc] processing block
+      # @return [Reel::Server::HTTP, Reel::Server::HTTPS]
+      def build_listener(&block)
+        if(args[:ssl])
+          ssl_config = Smash.new(args[:ssl][key].dup)
+          [:key, :cert].each do |key|
+            if(ssl_config[key])
+              ssl_config[key] = File.open(ssl_config.delete(key))
+            end
+          end
+          Reel::Server::HTTPS.supervise(args[:bind], args[:port], ssl_config, &block)
+        else
+          Reel::Server::HTTP.supervise(args[:bind], args[:port], &block)
+        end
+      end
+
+      # Size limit for inline body
+      BODY_TO_FILE_SIZE = 1024 * 10 # 10K
+
+      # Build message hash from request
+      #
+      # @param con [Reel::Connection]
+      # @param req [Reel::Request]
+      # @return [Hash]
+      # @note
+      #   if body size is greater than BODY_TO_FILE_SIZE
+      #   the body will be a temp file instead of a string
+      def build_message(con, req)
+        msg = Smash.new(
+          :request => req,
+          :headers => Smash[
+            req.headers.map{ |k,v| [k.downcase.tr('-', '_'), v]}
+          ],
+          :connection => con,
+          :query => parse_query_string(req.query_string),
+          :origin => req.remote_addr,
+          :authentication => {}
+        )
+        if(msg[:headers][:content_type] == 'application/json')
+          msg[:body] = MultiJson.load(
+            req.body.to_s
+          )
+        elsif(msg[:headers][:content_type] == 'application/x-www-form-urlencoded')
+          msg[:body] = parse_query_string(
+            req.body.to_s
+          )
+          if(msg[:body].size == 1 && msg[:body].values.first.is_a?(Array) && msg[:body].values.first.empty?)
+            msg[:body] = msg[:body].keys.first
+          end
+        elsif(msg[:headers][:content_length].to_i > BODY_TO_FILE_SIZE)
+          msg[:body] = Tempfile.new('carnivore-http')
+          while((chunk = req.body.readpartial(2048)))
+            msg[:body] << chunk
+          end
+          msg[:body].rewind
+        else
+          msg[:body] = req.body.to_s
+        end
+        if(msg[:headers][:authorization])
+          user, pass = Base64.urlsafe_decode64(
+            msg[:headers][:authorization].split(' ').last
+          ).split(':', 2)
+          msg[:authentication] = {
+            :username => user,
+            :password => pass
+          }
+        end
+        msg
+      end
+
+    end
+
+  end
+end

data/lib/carnivore-http/retry_delivery.rb

@@ -0,0 +1,88 @@
+require 'carnivore-http'
+
+module Carnivore
+  module Http
+
+    class RetryDelivery
+
+      include Celluloid
+      include Carnivore::Utils::Logging
+
+      # @return [String] message directory
+      attr_reader :message_directory
+
+      # Create new instance
+      #
+      # @param directory [String] path to messages
+      # @return [self]
+      def initialize(directory)
+        @message_directory = directory
+        every(60){ attempt_redelivery }
+      end
+
+      # Attempt to deliver messages found in message directory
+      #
+      # @return [TrueClass, FalseClass] attempt was made
+      # @note will not attempt if attempt is currently in progress
+      def attempt_redelivery(message_id = '*')
+        attempt = false
+        begin
+          unless(@delivering)
+            @delivering = true
+            attempt = true
+            Dir.glob(File.join(message_directory, "#{message_id}.json")).each do |file|
+              debug "Redelivery processing: #{file}"
+              begin
+                args = MultiJson.load(File.read(file)).to_smash
+                debug "Restored from file #{file}: #{args.inspect}"
+                if(redeliver(args[:message_id], args[:payload], args[:method], args[:url], args[:headers]))
+                  FileUtils.rm(file)
+                end
+              rescue => e
+                error "Failed to process file (#{file}): #{e.class}: #{e}"
+                debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+              end
+            end
+          end
+        rescue => e
+          error "Unexpected error encountered during message redelivery! #{e.class}: #{e}"
+          debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+        ensure
+          @delivering = false
+        end
+        attempt
+      end
+
+      # Attempt redelivery of message
+      #
+      # @param message [Carnivore::Message]
+      # @param payload [String] serialized payload
+      # @param method [Symbol] HTTP method (:get, :post, etc)
+      # @param url [String] endpoint URL
+      # @param headers [Hash] request headers
+      # @return [TrueClass, FalseClass] redelivery was successful
+      def redeliver(message_id, payload, method, url, headers)
+        begin
+          base = headers.empty? ? HTTP : HTTP.with_headers(headers)
+          uri = URI.parse(url)
+          if(uri.userinfo)
+            base = base.basic_auth(:user => uri.user, :pass => uri.password)
+          end
+          result = base.send(method, url, :body => payload)
+          if(result.code < 200 || result.code > 299)
+            error "Invalid response code received for #{message_id}: #{result.code} - #{result.reason}"
+            false
+          else
+            info "Successful delivery of message on retry! Message ID: #{message_id}"
+            true
+          end
+        rescue => e
+          error "Transmission redelivery failure (Message ID: #{message_id}) - #{e.class}: #{e}"
+          debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+          false
+        end
+      end
+
+    end
+  end
+end

data/lib/carnivore-http/version.rb

@@ -1,6 +1,6 @@
 module Carnivore
   module Http
     # current library version
-    VERSION = Gem::Version.new('0.1.8')
+    VERSION = Gem::Version.new('0.2.0')
   end
 end

data/lib/carnivore-http.rb

@@ -6,10 +6,13 @@ module Carnivore
   # HTTP namespace
   module Http
     autoload :PointBuilder, 'carnivore-http/point_builder'
+    autoload :RetryDelivery, 'carnivore-http/retry_delivery'
   end
 
   class Source
     autoload :Http, 'carnivore-http/http'
+    autoload :HttpSource, 'carnivore-http/http_source'
+    autoload :HttpPaths, 'carnivore-http/http_paths'
     autoload :HttpEndpoints, 'carnivore-http/http_endpoints'
   end
 
@@ -20,4 +23,5 @@ module Carnivore
 end
 
 Carnivore::Source.provide(:http, 'carnivore-http/http')
+Carnivore::Source.provide(:http_paths, 'carnivore-http/http_paths')
 Carnivore::Source.provide(:http_endpoints, 'carnivore-http/http_endpoints')

data/test/specs/http.rb

@@ -4,62 +4,65 @@ require 'carnivore-http'
 
 describe 'Carnivore::Source::Http' do
 
-  describe 'Building an HTTP based source' do
-
-    it 'returns the source' do
-      Carnivore::Source.build(
-        :type => :http,
-        :args => {
-          :name => :http_source,
-          :bind => '127.0.0.1',
-          :port => '8705'
-        }
-      )
-      t = Thread.new{ Carnivore.start! }
-      source_wait
-      Carnivore::Supervisor.supervisor[:http_source].wont_be_nil
-      t.terminate
+  before do
+    MessageStore.init
+    Carnivore::Source.build(
+      :type => :http,
+      :args => {
+        :name => :http_source,
+        :bind => '127.0.0.1',
+        :port => '8705'
+      }
+    ).add_callback(:store) do |message|
+      MessageStore.messages.push(message[:message][:body])
+      message.confirm!
     end
+    @runner = Thread.new{ Carnivore.start! }
+    source_wait
+  end
 
+  after do
+    @runner.terminate
   end
 
   describe 'HTTP source based communication' do
+
     before do
-      MessageStore.init
-      Carnivore::Source.build(
-        :type => :http,
-        :args => {
-          :name => :http_source,
-          :bind => '127.0.0.1',
-          :port => '8705'
-        }
-      ).add_callback(:store) do |message|
-        MessageStore.messages.push(message[:message][:body])
-      end
-      @runner = Thread.new{ Carnivore.start! }
-      source_wait
+      MessageStore.messages.clear
     end
 
-    after do
-      @runner.terminate
+    describe 'Building an HTTP based source' do
+
+      it 'returns the source' do
+        Carnivore::Supervisor.supervisor[:http_source].wont_be_nil
+      end
+
     end
 
     describe 'message transmissions' do
+
       it 'should accept message transmits' do
         Carnivore::Supervisor.supervisor[:http_source].transmit('test message')
       end
 
       it 'should receive messages' do
         Carnivore::Supervisor.supervisor[:http_source].transmit('test message 2')
-        source_wait
+        source_wait(2) do
+          !MessageStore.messages.empty?
+        end
+        MessageStore.messages.wont_be_empty
         MessageStore.messages.pop.must_equal 'test message 2'
       end
 
       it 'should accept http requests' do
         HTTP.get('http://127.0.0.1:8705/')
-        source_wait
+        source_wait(2) do
+          !MessageStore.messages.empty?
+        end
+        MessageStore.messages.wont_be_empty
         MessageStore.messages.pop.wont_be_nil
       end
+
     end
   end
 

data/test/specs/paths.rb

@@ -0,0 +1,87 @@
+require 'http'
+require 'minitest/autorun'
+require 'carnivore-http'
+
+
+describe 'Carnivore::Source::Http' do
+
+  before do
+    MessageStore.init
+
+    unless(@runner)
+      Carnivore::Source.build(
+        :type => :http_paths,
+        :args => {
+          :name => :fubar_source,
+          :path => '/fubar',
+          :method => :post,
+          :bind => '127.0.0.1',
+          :port => '8706'
+        }
+      ).add_callback(:store) do |message|
+        MessageStore.messages.push(message[:message][:body])
+        message.confirm!
+      end
+      Carnivore::Source.build(
+        :type => :http_paths,
+        :args => {
+          :name => :ohai_source,
+          :path => '/ohai',
+          :method => :get,
+          :bind => '127.0.0.1',
+          :port => '8706'
+        }
+      ).add_callback(:store) do |message|
+        MessageStore.messages.push(message[:message][:body])
+        message.confirm!
+      end
+      @runner = Thread.new{ Carnivore.start! }
+      source_wait
+    end
+  end
+
+  after do
+    @runner.terminate
+  end
+
+  describe 'HTTP source based communication' do
+
+    before do
+      MessageStore.messages.clear
+    end
+
+    describe 'Building an HTTP based source' do
+
+      it 'returns the sources' do
+        Carnivore::Supervisor.supervisor[:fubar_source].wont_be_nil
+        Carnivore::Supervisor.supervisor[:ohai_source].wont_be_nil
+      end
+
+    end
+
+    describe 'message transmissions' do
+
+      it 'should accept message transmits' do
+        Carnivore::Supervisor.supervisor[:fubar_source].transmit('test message')
+        Carnivore::Supervisor.supervisor[:ohai_source].transmit('test message')
+      end
+
+      it 'should receive messages' do
+        Carnivore::Supervisor.supervisor[:fubar_source].transmit('test message to fubar')
+        source_wait(4) do
+          !MessageStore.messages.empty?
+        end
+        MessageStore.messages.wont_be_empty
+        MessageStore.messages.pop.must_equal 'test message to fubar'
+        Carnivore::Supervisor.supervisor[:ohai_source].transmit('test message to ohai')
+        source_wait(4) do
+          !MessageStore.messages.empty?
+        end
+        MessageStore.messages.wont_be_empty
+        MessageStore.messages.pop.must_equal 'test message to ohai'
+      end
+
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: carnivore-http
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Chris Roberts
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-25 00:00:00.000000000 Z
+date: 2015-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: carnivore
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: htauth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Carnivore HTTP source
 email: chrisroberts.code@gmail.com
 executables: []
@@ -68,11 +82,15 @@ files:
 - lib/carnivore-http.rb
 - lib/carnivore-http/http.rb
 - lib/carnivore-http/http_endpoints.rb
+- lib/carnivore-http/http_paths.rb
+- lib/carnivore-http/http_source.rb
 - lib/carnivore-http/point_builder.rb
+- lib/carnivore-http/retry_delivery.rb
 - lib/carnivore-http/utils.rb
 - lib/carnivore-http/version.rb
 - test/spec.rb
 - test/specs/http.rb
+- test/specs/paths.rb
 homepage: https://github.com/carnivore-rb/carnivore-http
 licenses:
 - Apache 2.0