lifter 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f28fdb4670f7959c6eef5fff3ac85894c016293
-  data.tar.gz: 763a8953d4ba2ed82f969cd28c5cf2150dd0dc81
+  metadata.gz: 795dbedbc9fb64ba72602c68ca1eb1352a1e135d
+  data.tar.gz: 6a4c454a3a9c5005f393c3de0a17f5797e50cf3d
 SHA512:
-  metadata.gz: b3f02a5c37031f345a3a3320ce4155f67fab09f40dcd98d91a5b12b87a081fbad616416779c9ff47a65ae0a4c1b264dc8d746cfff3a43140243d88676925c330
-  data.tar.gz: ce2838c2521b7aa19926af9e589199fd51c7158e6fb96618cf7f04d62f7309b07824e76ece43afe277f3828db41668e8eed843d1ce3e5091d452ab5516ba0deb
+  metadata.gz: 2b8879248daabb268a9b0ac9bafcea9bf752ad1253d7293374b01d81e5a8b50d11d969f7a7123df42314b4193821b6e883c2b39b08a70c2b86a3321cd6fb9752
+  data.tar.gz: f10d0d10f850e39471e7bf001cc898db9dee34a21ba22fc77348fb143b612d5e6352b6e36e2bfb6c6fb2897da839263b246d9229df3433cd2ed711e206605a49

data/lib/lifter.rb

@@ -1,6 +1,7 @@
 require 'multipart_parser/parser'
 require 'multipart_parser/reader'
 
+require 'lifter/errors'
 require 'lifter/thread_pool'
 require 'lifter/file_pool'
 require 'lifter/webhook'

data/lib/lifter/connection.rb

@@ -3,9 +3,10 @@ require 'http/parser'
 
 module Lifter
   class Connection < EventMachine::Connection
-    Request = Struct.new(:headers, :params, :remote_ip)
+    Request = Struct.new(:headers, :params, :remote_ip, :origin)
 
     PAYLOAD_METHODS = ['put', 'post'].freeze
+    OPTIONS_METHOD = 'options'.freeze
     CONTENT_TYPE_KEY = 'content-type'.freeze
     MULTIPART_CONTENT_TYPE = 'multipart/form-data'.freeze
 
@@ -16,6 +17,7 @@ module Lifter
 
       @is_multipart = nil
       @has_payload = nil
+      @options_request = nil
 
       @multipart_reader = nil
       @inline_reader = nil
@@ -23,6 +25,8 @@ module Lifter
       @server = nil
       @request = Request.new
 
+      @gracefully_close = false
+
       @parser = HTTP::Parser.new
 
       @parser.on_message_begin = proc do
@@ -31,7 +35,16 @@ module Lifter
 
       @parser.on_headers_complete = proc do
         process_headers
-        start_payload if payload?
+
+        if payload?
+          start_payload
+        else
+          if request_method?(:options)
+            handle_options_request
+          else
+            close(gracefully: true)
+          end
+        end
       end
 
       @parser.on_body = proc do |data|
@@ -60,7 +73,24 @@ module Lifter
     end
 
     def unbind
+      @payload.cancel if !gracefully_close? && !@payload.nil?
+    end
+
+    def close(opts = {})
+      @gracefully_close = opts[:gracefully] == true
+
+      EventMachine.next_tick do
+        close_connection(true)
+      end
+    end
+
+    def cancel
       @payload.cancel if !@payload.nil?
+      close(gracefully: true)
+    end
+
+    def gracefully_close?
+      @gracefully_close == true
     end
 
     def http_version
@@ -71,16 +101,23 @@ module Lifter
       '127.0.0.1'
     end
 
-    def respond(code, status)
+    def respond(code, status, headers = {}, body = '')
+      return if gracefully_close?
+
       EventMachine.next_tick do
         response = "HTTP/#{http_version.join('.')} #{code} #{status}"
-        send_data(response)
-      end
-    end
 
-    def close
-      EventMachine.next_tick do
-        close_connection(true)
+        headers = {} if headers.empty?
+        headers[:content_length] = body.bytesize if headers[:content_length].nil?
+        headers[:access_control_allow_origin] = @request.origin if !@request.origin.nil?
+
+        formatted_headers = format_response_headers(headers)
+
+        response << "\r\n" << formatted_headers
+        response << "\r\n\r\n"
+        response << body
+
+        send_data(response)
       end
     end
 
@@ -95,9 +132,6 @@ module Lifter
     end
 
     private def finish_request
-      clear_request
-      clear_multipart
-      clear_payload
     end
 
     private def start_payload
@@ -123,6 +157,8 @@ module Lifter
       @request.headers = headers
 
       @request.remote_ip = remote_ip
+
+      @request.origin = @request.headers['origin']
     end
 
     private def payload?
@@ -133,6 +169,10 @@ module Lifter
       @has_payload
     end
 
+    private def request_method?(method)
+      @parser.http_method.to_s.downcase == method.to_s.downcase
+    end
+
     private def clear_multipart
       @is_multipart = nil
     end
@@ -166,5 +206,27 @@ module Lifter
     private def start_inline_payload
       @payload = Payloads::InlinePayload.new(self, file_manager)
     end
+
+    private def handle_options_request
+      cors_headers = {
+        access_control_allow_methods: 'PUT, POST',
+        access_control_allow_headers: 'accept, content-type',
+      }
+
+      respond(200, 'OK', cors_headers)
+    end
+
+    private def format_response_headers(headers = {})
+      formatted_headers = []
+
+      headers.each_pair do |key, value|
+        formatted_key = key.to_s.strip.downcase.gsub(/[:_\- ]/, '-').squeeze('-').gsub(/[\s]+/, '')
+        formatted_key = formatted_key.split('-').map { |s| s.capitalize }.join('-')
+
+        formatted_headers << "#{formatted_key}: #{value}"
+      end
+
+      formatted_headers.join("\r\n")
+    end
   end
 end

data/lib/lifter/errors.rb

@@ -0,0 +1,6 @@
+module Lifter
+  module Errors
+    class WebhookFailed < StandardError; end
+    class InvalidWebhookMethod < StandardError; end
+  end
+end

data/lib/lifter/file_manager.rb

@@ -43,10 +43,7 @@ module Lifter
         file.write(data)
 
         if file.prologue.length >= @upload_prologue_size
-          @webhooks.push(file_id) do
-            webhook = create_authorize_webhook(connection, file)
-            webhook.deliver
-          end
+          authorize_file(connection, file_id, file)
         end
       end
     end
@@ -60,21 +57,13 @@ module Lifter
 
         file.mv("#{@working_dir}/completed/#{file_id}")
 
-        @files.remove(file_id)
-
         # In the event the upload was too small for the prologue size to have been met previously,
-        # ensure the authorize webhook is fired off before completed.
+        # ensure the authorize webhook is fired off before the completed webhook.
         if file.prologue.length < @upload_prologue_size
-          @webhooks.push(file_id) do
-            webhook = create_authorize_webhook(connection, file)
-            webhook.deliver
-          end
+          authorize_file(connection, file_id, file)
         end
 
-        @webhooks.push(file_id) do
-          webhook = create_completed_webhook(connection, file)
-          webhook.deliver
-        end
+        complete_file(connection, file_id, file)
       end
     end
 
@@ -101,18 +90,28 @@ module Lifter
       end
     end
 
-    private def create_authorize_webhook(connection, file)
+    private def authorize_file(connection, file_id, file)
+      return if file.authorized? || file.pending_authorization?
+
+      @webhooks.push(file_id) do
+        webhook = create_authorize_webhook(connection, file_id, file)
+        file.pending_authorization
+        webhook.deliver
+      end
+    end
+
+    private def create_authorize_webhook(connection, file_id, file)
       webhook = Webhook.new(@authorize_webhook_endpoint)
 
       headers = file.original_request.headers.dup
       params = file.original_request.params.dup
 
-      headers = clean_request_headers(headers)
+      headers = scrub_request_headers(headers)
       headers['x-upload-ip'] = file.original_request.remote_ip
 
       params[file.param] = {
         file_name: file.original_name,
-        file_prologue: file.prologue.length
+        file_prologue: file.prologue
       }
 
       webhook.headers = headers
@@ -120,18 +119,30 @@ module Lifter
 
       webhook.on_failure do
         connection.cancel
+        cancel_file(connection, file_id)
+      end
+
+      webhook.on_success do
+        file.authorize
       end
 
       webhook
     end
 
-    private def create_completed_webhook(connection, file)
+    private def complete_file(connection, file_id, file)
+      @webhooks.push(file_id) do
+        webhook = create_completed_webhook(connection, file_id, file)
+        webhook.deliver
+      end
+    end
+
+    private def create_completed_webhook(connection, file_id, file)
       webhook = Webhook.new(@completed_webhook_endpoint)
 
       headers = file.original_request.headers.dup
       params = file.original_request.params.dup
 
-      headers = clean_request_headers(headers)
+      headers = scrub_request_headers(headers)
       headers['x-upload-ip'] = file.original_request.remote_ip
 
       params[file.param] = {
@@ -140,14 +151,23 @@ module Lifter
         file_hash: file.hash
       }
 
+      webhook.headers = headers
+      webhook.params = params
+
       webhook.on_failure do
         connection.cancel
+        @files.remove(file_id)
+      end
+
+      webhook.on_success do
+        connection.respond(200, 'OK')
+        @files.remove(file_id)
       end
 
       webhook
     end
 
-    private def clean_request_headers(headers)
+    private def scrub_request_headers(headers)
       SCRUB_HEADERS.each do |header_key|
         headers.delete(header_key)
       end

data/lib/lifter/file_upload.rb

@@ -10,7 +10,13 @@ module Lifter
     attr_reader :prologue, :original_request, :original_name, :param
 
     def initialize(file, opts = {})
+      @mutex = Mutex.new
+
       @file = file
+      @path = file.path
+
+      @authorized = false
+      @pending_authorization = false
 
       @hash = setup_hash(opts[:hash_method])
 
@@ -37,25 +43,59 @@ module Lifter
     end
 
     def close
-      @file.close
+      @file.close if !@file.closed?
     end
 
     def rm
       FileUtils.rm(full_path)
+      @path = nil
     end
 
     def mv(new_path)
       FileUtils.mv(full_path, new_path)
+      @path = new_path
     end
 
     def full_path
-      File.expand_path(@file.path)
+      File.expand_path(@path)
     end
 
     def hash
       @hash.hexdigest
     end
 
+    def authorize
+      @mutex.synchronize do
+        @authorized = true
+      end
+    end
+
+    def pending_authorization
+      @mutex.synchronize do
+        @pending_authorization = true
+      end
+    end
+
+    def pending_authorization?
+      pending = false
+
+      @mutex.synchronize do
+        pending = @pending_authorization == true
+      end
+
+      pending
+    end
+
+    def authorized?
+      authorized = false
+
+      @mutex.synchronize do
+        authorized = @authorized == true
+      end
+
+      authorized
+    end
+
     private def setup_hash(hash_type)
       case hash_type
       when :md5

data/lib/lifter/payloads/multipart_payload.rb

@@ -23,7 +23,7 @@ module Lifter
         return if !current_part?
 
         if current_part.type == :file
-          @file_manager.cancel_file(@current_part.id)
+          @file_manager.cancel_file(@connection, @current_part.id)
         end
 
         @current_part = nil
@@ -69,7 +69,7 @@ module Lifter
 
         @reader.on_end do
           @connection.respond(200, 'OK')
-          @connection.close
+          @connection.close(gracefully: true)
         end
 
         @reader.on_error do |message|

data/lib/lifter/thread_pool.rb

@@ -97,9 +97,6 @@ module Lifter
             rescue StandardError => e
               puts e.to_s
               puts e.backtrace
-              exit
-              add_pending(job_tag)
-              queue.push([job_tag, job])
             end
           end
         end

data/lib/lifter/version.rb

@@ -1,3 +1,3 @@
 module Lifter
-  VERSION = "0.1.0".freeze
+  VERSION = "0.1.1".freeze
 end

data/lib/lifter/webhook.rb

@@ -2,6 +2,32 @@ require 'http'
 
 module Lifter
   class Webhook
+    # Courtesy of: http://dev.mensfeld.pl/2012/01/converting-nested-hash-into-http-url-params-hash-version-in-ruby/
+    module ParamNester
+      def self.encode(value, key = nil, out_hash = {})
+        case value
+        when Hash
+          value.each { |k,v| encode(v, append_key(key, k), out_hash) }
+          out_hash
+        when Array
+          value.each { |v| encode(v, "#{key}[]", out_hash) }
+          out_hash
+        when nil
+          ''
+        else
+          out_hash[key] = value
+          out_hash
+        end
+      end
+
+      def self.append_key(root_key, key)
+        root_key.nil? ? :"#{key}" : :"#{root_key}[#{key.to_s}]"
+      end
+    end
+
+    RETRY_CODES = [500, 502, 503, 504]
+    RETRY_LIMIT = 3
+
     attr_reader :url, :method, :headers, :params
 
     def initialize(endpoint)
@@ -9,14 +35,24 @@ module Lifter
       @method = endpoint.method
       @headers = {}
       @params = {}
+
+      @retry_count = 0
+      @retry_limit = RETRY_LIMIT
+
+      @on_success = nil
+      @on_failure = nil
     end
 
     def headers=(headers)
       @headers = headers
     end
 
-    def params=(params)
-      @params = params
+    def params=(params = {})
+      @params = ParamNester.encode(params)
+    end
+
+    def on_success(&block)
+      @on_success = block
     end
 
     def on_failure(&block)
@@ -24,6 +60,33 @@ module Lifter
     end
 
     def deliver
+      begin
+        start_delivery
+      rescue Errors::WebhookFailed => e
+        @on_failure.call if !@on_failure.nil?
+      end
+
+      @on_success.call if !@on_success.nil?
+    end
+
+    private def start_delivery
+      begin
+        response = finish_delivery
+      rescue StandardError => e
+        raise Errors::WebhookFailed.new
+      end
+
+      return if response.code == 200
+
+      if RETRY_CODES.include?(response.code) && @retry_count < @retry_limit
+        @retry_count += 1
+        start_delivery
+      else
+        raise Errors::WebhookFailed.new
+      end
+    end
+
+    private def finish_delivery
       http_stub = HTTP.headers(@headers)
 
       case @method.to_sym
@@ -34,7 +97,7 @@ module Lifter
       when :put
         http_stub.put(@url, form: @params)
       else
-        raise StandardError.new('unsupported http method in webhook')
+        raise Errors::InvalidWebhookMethod.new
       end
     end
   end

data/test/test.rb

@@ -1,3 +1,5 @@
+lib = File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'lifter'
 
 server = Lifter::Server.new do |config|
@@ -13,14 +15,14 @@ server = Lifter::Server.new do |config|
   working_dir 'tmp/uploads'
 
   # Define maximum size in bytes of a file upload. Files larger than this will be automatically
-  # removed, the connection closed, and no uploaded webhook will fire.
+  # removed, the connection closed, and no completed webhook will fire.
   #
   max_upload_size 500 * 1024 * 1024
 
   # Specify desired digest type for file uploads. Passed in uploaded webhook after upload completes.
   # Possible options: md5, sha1, sha256, sha512.
   #
-  upload_hash_method :sha1
+  upload_hash_method :sha256
 
   # Configure maximum number of bytes to pass along in authorize webhook.
   #
@@ -38,7 +40,7 @@ server = Lifter::Server.new do |config|
   # as the first <prologue_limit> bytes of data is received. Non-200 responses for one part will not
   # remove data from other parts, although the connection will still be terminated.
   #
-  authorize_webhook :post, 'http://127.0.0.1:8081/uploads/authorize'
+  authorize_webhook :post, 'http://127.0.0.1:3000/uploads/authorize'
 
   # A request to this webhook is made once a single file upload completes. In the event the upload
   # is multipart with multiple files, this endpoint will be called once for each file, upon
@@ -46,7 +48,7 @@ server = Lifter::Server.new do |config|
   #
   # An authorize webhook is always sent prior to sending this webhook.
   #
-  completed_webhook :post, 'http://127.0.0.1:8081/uploads/ingest'
+  completed_webhook :post, 'http://127.0.0.1:3000/uploads/complete'
 end
 
 server.start

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lifter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Michael Amundson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-20 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
@@ -80,6 +80,7 @@ files:
 - lib/lifter.rb
 - lib/lifter/config.rb
 - lib/lifter/connection.rb
+- lib/lifter/errors.rb
 - lib/lifter/file_manager.rb
 - lib/lifter/file_pool.rb
 - lib/lifter/file_upload.rb
@@ -115,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Painless file uploads