lifter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8f28fdb4670f7959c6eef5fff3ac85894c016293
+  data.tar.gz: 763a8953d4ba2ed82f969cd28c5cf2150dd0dc81
+SHA512:
+  metadata.gz: b3f02a5c37031f345a3a3320ce4155f67fab09f40dcd98d91a5b12b87a081fbad616416779c9ff47a65ae0a4c1b264dc8d746cfff3a43140243d88676925c330
+  data.tar.gz: ce2838c2521b7aa19926af9e589199fd51c7158e6fb96618cf7f04d62f7309b07824e76ece43afe277f3828db41668e8eed843d1ce3e5091d452ab5516ba0deb

data/.gitignore

@@ -0,0 +1,5 @@
+tmp/
+design.txt
+*.gem
+.DS_Store
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Michael Amundson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/lifter/config.rb

@@ -0,0 +1,54 @@
+module Lifter
+  class Config
+    Webhook = Struct.new(:method, :url)
+
+    def initialize(&definition)
+      # Defaults
+      @config = {
+        host: '127.0.0.1'
+      }
+
+      instance_eval(&definition)
+    end
+
+    def get(key)
+      key = key.to_sym
+
+      raise ArgumentError.new('unknown key') if !@config.has_key?(key)
+
+      @config[key]
+    end
+
+    def host(host)
+      @config[:host] = host
+    end
+
+    def port(port)
+      @config[:port] = port.to_i
+    end
+
+    def working_dir(path)
+      @config[:working_dir] = path
+    end
+
+    def upload_hash_method(upload_hash_method)
+      @config[:upload_hash_method] = upload_hash_method
+    end
+
+    def max_upload_size(max_upload_size)
+      @config[:max_upload_size] = max_upload_size
+    end
+
+    def upload_prologue_size(upload_prologue_size)
+      @config[:upload_prologue_size] = upload_prologue_size
+    end
+
+    def authorize_webhook(method, url)
+      @config[:authorize_webhook] = Webhook.new(method, url)
+    end
+
+    def completed_webhook(method, url)
+      @config[:completed_webhook] = Webhook.new(method, url)
+    end
+  end
+end

data/lib/lifter/connection.rb

@@ -0,0 +1,170 @@
+require 'eventmachine'
+require 'http/parser'
+
+module Lifter
+  class Connection < EventMachine::Connection
+    Request = Struct.new(:headers, :params, :remote_ip)
+
+    PAYLOAD_METHODS = ['put', 'post'].freeze
+    CONTENT_TYPE_KEY = 'content-type'.freeze
+    MULTIPART_CONTENT_TYPE = 'multipart/form-data'.freeze
+
+    attr_reader :request
+
+    def initialize
+      super
+
+      @is_multipart = nil
+      @has_payload = nil
+
+      @multipart_reader = nil
+      @inline_reader = nil
+
+      @server = nil
+      @request = Request.new
+
+      @parser = HTTP::Parser.new
+
+      @parser.on_message_begin = proc do
+        start_request
+      end
+
+      @parser.on_headers_complete = proc do
+        process_headers
+        start_payload if payload?
+      end
+
+      @parser.on_body = proc do |data|
+        receive_payload_data(data) if payload?
+      end
+
+      @parser.on_message_complete = proc do
+        finish_request
+      end
+    end
+
+    def server=(server)
+      raise ArgumentError.new('incorrect type') if !server.is_a?(Server)
+
+      @server = server
+    end
+
+    def file_manager
+      raise StandardError.new('server not defined') if @server.nil?
+
+      @server.file_manager
+    end
+
+    def receive_data(data)
+      @parser << data
+    end
+
+    def unbind
+      @payload.cancel if !@payload.nil?
+    end
+
+    def http_version
+      @parser.http_version || [1, 0]
+    end
+
+    def remote_ip
+      '127.0.0.1'
+    end
+
+    def respond(code, status)
+      EventMachine.next_tick do
+        response = "HTTP/#{http_version.join('.')} #{code} #{status}"
+        send_data(response)
+      end
+    end
+
+    def close
+      EventMachine.next_tick do
+        close_connection(true)
+      end
+    end
+
+    private def receive_payload_data(data)
+      @payload << data
+    end
+
+    private def start_request
+      clear_request
+      clear_multipart
+      clear_payload
+    end
+
+    private def finish_request
+      clear_request
+      clear_multipart
+      clear_payload
+    end
+
+    private def start_payload
+      if multipart?
+        start_multipart_payload
+      else
+        start_inline_payload
+      end
+    end
+
+    private def clear_request
+      @request = Request.new
+    end
+
+    private def process_headers
+      headers = {}
+
+      @parser.headers.each_pair do |key, value|
+        normalized_key = key.strip.downcase
+        headers[normalized_key] = value
+      end
+
+      @request.headers = headers
+
+      @request.remote_ip = remote_ip
+    end
+
+    private def payload?
+      return @has_payload if !@has_payload.nil?
+
+      @has_payload = PAYLOAD_METHODS.include?(@parser.http_method.to_s.downcase)
+
+      @has_payload
+    end
+
+    private def clear_multipart
+      @is_multipart = nil
+    end
+
+    private def multipart?
+      return @is_multipart if !@is_multipart.nil?
+
+      content_type = @request.headers[CONTENT_TYPE_KEY]
+
+      if content_type.nil? || content_type.empty?
+        @is_multipart = false
+      else
+        @is_multipart = content_type.split(';').first.downcase == MULTIPART_CONTENT_TYPE
+      end
+
+      @is_multipart
+    end
+
+    private def clear_payload
+      @has_payload = nil
+      @payload = nil
+    end
+
+    private def start_multipart_payload
+      content_type = @request.headers[CONTENT_TYPE_KEY]
+      multipart_boundary = MultipartParser::Reader.extract_boundary_value(content_type)
+
+      @payload = Payloads::MultipartPayload.new(self, file_manager, multipart_boundary)
+    end
+
+    private def start_inline_payload
+      @payload = Payloads::InlinePayload.new(self, file_manager)
+    end
+  end
+end

data/lib/lifter/file_manager.rb

@@ -0,0 +1,158 @@
+require 'fileutils'
+
+module Lifter
+  class FileManager
+    DEFAULT_HASH_METHOD = :md5
+
+    SCRUB_HEADERS = %w(host content-type content-length accept accept-encoding accept-language
+      connection)
+
+    def initialize(config)
+      @working_dir = resolve_working_dir(config.get(:working_dir))
+
+      @authorize_webhook_endpoint = config.get(:authorize_webhook)
+      @completed_webhook_endpoint = config.get(:completed_webhook)
+
+      @work = ThreadPool.new(5)
+      @webhooks = ThreadPool.new(5)
+      @files = FilePool.new
+
+      @upload_hash_method = config.get(:upload_hash_method) || DEFAULT_HASH_METHOD
+      @upload_prologue_size = config.get(:upload_prologue_size)
+    end
+
+    def open_file(connection, file_id, file_param, file_name)
+      @work.push(file_id) do
+        file = File.open("#{@working_dir}/progress/#{file_id}", 'wb')
+
+        file_opts = {
+          hash_method: @upload_hash_method,
+          prologue_size: @upload_prologue_size,
+          original_name: file_name,
+          original_request: connection.request,
+          param: file_param
+        }
+
+        @files.add(file_id, file, file_opts)
+      end
+    end
+
+    def write_file_data(connection, file_id, data)
+      @work.push(file_id) do
+        file = @files.get(file_id)
+        file.write(data)
+
+        if file.prologue.length >= @upload_prologue_size
+          @webhooks.push(file_id) do
+            webhook = create_authorize_webhook(connection, file)
+            webhook.deliver
+          end
+        end
+      end
+    end
+
+    def close_file(connection, file_id)
+      @work.push(file_id) do
+        file = @files.get(file_id)
+
+        file.flush
+        file.close
+
+        file.mv("#{@working_dir}/completed/#{file_id}")
+
+        @files.remove(file_id)
+
+        # In the event the upload was too small for the prologue size to have been met previously,
+        # ensure the authorize webhook is fired off before completed.
+        if file.prologue.length < @upload_prologue_size
+          @webhooks.push(file_id) do
+            webhook = create_authorize_webhook(connection, file)
+            webhook.deliver
+          end
+        end
+
+        @webhooks.push(file_id) do
+          webhook = create_completed_webhook(connection, file)
+          webhook.deliver
+        end
+      end
+    end
+
+    def cancel_file(connection, file_id)
+      @webhooks.clear(file_id)
+
+      @work.push(file_id) do
+        file = @files.get(file_id)
+
+        file.close
+        file.rm
+
+        @files.remove(file_id)
+      end
+    end
+
+    private def resolve_working_dir(working_dir)
+      if working_dir.nil?
+        Dir.pwd
+      elsif working_dir[0, 1] == '/'
+        working_dir
+      else
+        "#{Dir.pwd}/#{working_dir}"
+      end
+    end
+
+    private def create_authorize_webhook(connection, file)
+      webhook = Webhook.new(@authorize_webhook_endpoint)
+
+      headers = file.original_request.headers.dup
+      params = file.original_request.params.dup
+
+      headers = clean_request_headers(headers)
+      headers['x-upload-ip'] = file.original_request.remote_ip
+
+      params[file.param] = {
+        file_name: file.original_name,
+        file_prologue: file.prologue.length
+      }
+
+      webhook.headers = headers
+      webhook.params = params
+
+      webhook.on_failure do
+        connection.cancel
+      end
+
+      webhook
+    end
+
+    private def create_completed_webhook(connection, file)
+      webhook = Webhook.new(@completed_webhook_endpoint)
+
+      headers = file.original_request.headers.dup
+      params = file.original_request.params.dup
+
+      headers = clean_request_headers(headers)
+      headers['x-upload-ip'] = file.original_request.remote_ip
+
+      params[file.param] = {
+        file_name: file.original_name,
+        file_path: file.full_path,
+        file_hash: file.hash
+      }
+
+      webhook.on_failure do
+        connection.cancel
+      end
+
+      webhook
+    end
+
+    private def clean_request_headers(headers)
+      SCRUB_HEADERS.each do |header_key|
+        headers.delete(header_key)
+      end
+
+      headers
+    end
+  end
+end

data/lib/lifter/file_pool.rb

@@ -0,0 +1,20 @@
+module Lifter
+  class FilePool
+    def initialize
+      @files = {}
+    end
+
+    def get(file_id)
+      @files[file_id]
+    end
+
+    def add(file_id, file, opts = {})
+      file_upload = FileUpload.new(file, opts)
+      @files[file_id] = file_upload
+    end
+
+    def remove(file_id)
+      @files.delete(file_id)
+    end
+  end
+end

data/lib/lifter/file_upload.rb

@@ -0,0 +1,74 @@
+require 'fileutils'
+require 'digest/sha1'
+require 'digest/sha2'
+require 'digest/md5'
+
+module Lifter
+  class FileUpload
+    DEFAULT_PROLOGUE_SIZE = 10 * 1024
+
+    attr_reader :prologue, :original_request, :original_name, :param
+
+    def initialize(file, opts = {})
+      @file = file
+
+      @hash = setup_hash(opts[:hash_method])
+
+      @prologue_limit = opts[:prologue_size] || DEFAULT_PROLOGUE_SIZE
+      @prologue = ''
+
+      @original_request = opts[:original_request]
+      @original_name = opts[:original_name]
+      @param = opts[:param]
+    end
+
+    def write(data)
+      @file.write(data)
+
+      @hash << data
+
+      if @prologue.length < @prologue_limit
+        @prologue << data[0, @prologue_limit - @prologue.length]
+      end
+    end
+
+    def flush
+      @file.flush
+    end
+
+    def close
+      @file.close
+    end
+
+    def rm
+      FileUtils.rm(full_path)
+    end
+
+    def mv(new_path)
+      FileUtils.mv(full_path, new_path)
+    end
+
+    def full_path
+      File.expand_path(@file.path)
+    end
+
+    def hash
+      @hash.hexdigest
+    end
+
+    private def setup_hash(hash_type)
+      case hash_type
+      when :md5
+        Digest::MD5.new
+      when :sha1
+        Digest::SHA1.new
+      when :sha256
+        Digest::SHA256.new
+      when :sha512
+        Digest::SHA512.new
+      else
+        Digest::MD5.new
+      end
+    end
+  end
+end

data/lib/lifter/payloads/inline_payload.rb

@@ -0,0 +1,10 @@
+module Lifter
+  module Payloads
+    class InlinePayload
+      def initialize(connection, file_manager)
+        @connection = connection
+        @file_manager = file_manager
+      end
+    end
+  end
+end

data/lib/lifter/payloads/multipart_payload.rb

@@ -0,0 +1,85 @@
+module Lifter
+  module Payloads
+    class MultipartPayload
+      CurrentPart = Struct.new(:id, :type, :name)
+
+      def initialize(connection, file_manager, multipart_boundary)
+        @connection = connection
+        @file_manager = file_manager
+        @reader = MultipartParser::Reader.new(multipart_boundary)
+
+        @current_part = nil
+
+        @params = {}
+
+        setup_callbacks
+      end
+
+      def <<(data)
+        @reader.write(data)
+      end
+
+      def cancel
+        return if !current_part?
+
+        if current_part.type == :file
+          @file_manager.cancel_file(@current_part.id)
+        end
+
+        @current_part = nil
+      end
+
+      def current_part?
+        !@current_part.nil?
+      end
+
+      private def setup_callbacks
+        @reader.on_part do |part|
+          @current_part = CurrentPart.new
+
+          @current_part.id = SecureRandom.hex(10)
+          @current_part.name = part.name
+
+          if part.filename.nil?
+            @current_part.type = :param
+            @params[@current_part.name] = ''
+          else
+            @current_part.type = :file
+            @file_manager.open_file(@connection, @current_part.id, @current_part.name, part.filename)
+          end
+
+          part.on_data do |data|
+            if @current_part.type == :param
+              @params[@current_part.name] << data
+            else
+              @file_manager.write_file_data(@connection, @current_part.id, data)
+            end
+          end
+
+          part.on_end do
+            @connection.request.params = @params
+
+            if @current_part.type == :file
+              @file_manager.close_file(@connection, @current_part.id)
+            end
+
+            @current_part = nil
+          end
+        end
+
+        @reader.on_end do
+          @connection.respond(200, 'OK')
+          @connection.close
+        end
+
+        @reader.on_error do |message|
+          if !@current_part.nil? && @current_part.type == :file
+            @file_manager.cancel_file(@connection, @current_part.id)
+          end
+
+          @current_part = nil
+        end
+      end
+    end
+  end
+end

data/lib/lifter/server.rb

@@ -0,0 +1,26 @@
+require 'eventmachine'
+
+module Lifter
+  class Server
+    attr_reader :config, :file_manager
+
+    def initialize(&config)
+      @config = Config.new(&config)
+      @file_manager = FileManager.new(@config)
+    end
+
+    def start
+      EventMachine.epoll if EventMachine.epoll?
+      EventMachine.kqueue if EventMachine.kqueue?
+
+      EventMachine.run do
+        host = @config.get(:host)
+        port = @config.get(:port)
+
+        EventMachine.start_server(host, port, Connection) do |connection|
+          connection.server = self
+        end
+      end
+    end
+  end
+end