pantry 0.0.0 → 0.1.0

data/lib/pantry/communication/file_service/file_progress.rb

@@ -0,0 +1,164 @@
+module Pantry
+  module Communication
+    class FileService
+
+      # Informational object for keeping track of file upload progress and
+      # important information.
+      class UploadInfo
+
+        # Identity of the Receiver we're sending a file to
+        attr_accessor :receiver_uuid
+
+        # The file session identity from the Receiver
+        attr_accessor :file_uuid
+
+        def initialize
+          @finish_future = Celluloid::Future.new
+        end
+
+        # Block and wait for the file upload to finish
+        def wait_for_finish(timeout = nil)
+          @finish_future.value(timeout)
+        end
+
+        def finished!
+          @finish_future.signal(OpenStruct.new(:value => self))
+        end
+
+      end
+
+      # Sending-side version of UploadInfo
+      class SendingFile < UploadInfo
+        attr_reader :path, :file
+
+        def initialize(file_path, receiver_uuid, file_uuid)
+          super()
+          @path = file_path
+          @file_uuid = file_uuid
+          @file = File.open(@path, "r")
+
+          @receiver_uuid = receiver_uuid
+
+          @file_size = @file.size
+          @total_bytes_sent = 0
+
+          Pantry.ui.progress_start(@file_size)
+        end
+
+        def read(offset, bytes_to_read)
+          @total_bytes_sent += bytes_to_read
+          Pantry.ui.progress_step(bytes_to_read)
+
+          @file.seek(offset)
+          @file.read(bytes_to_read)
+        end
+
+        def finished!
+          Pantry.ui.progress_finish
+
+          @file.close
+          super
+        end
+
+        def finished?
+          @total_bytes_sent == @file_size || @file.closed?
+        end
+
+      end
+
+      # Receiving-side version of UploadInfo
+      # Can be configured with a completion block that will be executed once the
+      # file has been fully received and checksum verified.
+      class ReceivingFile < UploadInfo
+
+        # Location of the tempfile containing the contents of the uploaded file
+        attr_reader :uploaded_path
+
+        attr_reader :file_size, :checksum, :uploaded_path
+        attr_accessor :sender_uuid
+
+        def initialize(file_size, checksum, chunk_size, pipeline_size)
+          super()
+          @file_uuid = SecureRandom.uuid
+          @file_size = file_size
+          @checksum  = checksum
+
+          @chunk_size    = chunk_size
+          @pipeline_size = pipeline_size
+
+          @uploaded_file = Tempfile.new(file_uuid)
+          @uploaded_path = @uploaded_file.path
+
+          @next_requested_file_offset = 0
+          @current_pipeline_size      = 0
+
+          @chunk_count      = (@file_size.to_f / @chunk_size.to_f).ceil
+          @requested_chunks = 0
+          @received_chunks  = 0
+        end
+
+        def on_complete(&block)
+          @completion_block = block
+        end
+
+        def chunks_to_fetch(&block)
+          chunks_to_fill_pipeline = [
+            (@pipeline_size - @current_pipeline_size),
+            @chunk_count - @requested_chunks
+          ].min
+
+          chunks_to_fill_pipeline.times do
+            block.call(@next_requested_file_offset, @chunk_size)
+
+            @next_requested_file_offset += @chunk_size
+            @current_pipeline_size += 1
+            @requested_chunks      += 1
+          end
+        end
+
+        def write_chunk(offset, size, data)
+          @current_pipeline_size -= 1
+          @received_chunks       += 1
+
+          @uploaded_file.seek(offset)
+          @uploaded_file.write(data)
+
+          if @received_chunks == @chunk_count
+            @uploaded_file.close
+          end
+        end
+
+        def finished!
+          @uploaded_file.close
+
+          if @completion_block && valid?
+            begin
+              @completion_block.call
+            rescue => ex
+              Pantry.logger.debug("[Receive File] Error running completion block #{ex.inspect}")
+            end
+          end
+
+          super
+        end
+
+        def complete?
+          @uploaded_file.closed?
+        end
+        alias finished? complete?
+
+        def valid?
+          return @is_valid if defined?(@is_valid)
+          uploaded_checksum = Pantry.file_checksum(@uploaded_file.path)
+          @is_valid = (uploaded_checksum == @checksum)
+        end
+
+        def remove
+          @uploaded_file.unlink
+        end
+
+      end
+
+    end
+  end
+end

data/lib/pantry/communication/file_service/receive_file.rb

@@ -0,0 +1,97 @@
+module Pantry
+  module Communication
+
+    # Chunk file receiving tool that implements the protocol as described here
+    #  http://zguide.zeromq.org/page:all#Transferring-Files
+    #
+    # In short, this tool requests chunks in a pipeline flow, writing out
+    # the received chunks to the file system at the given path.
+    class FileService::ReceiveFile
+      include Celluloid
+
+      attr_accessor :pipeline_size, :chunk_size
+
+      def initialize(service, chunk_size: 250_000, pipeline_size: 10)
+        @service = service
+
+        @chunk_size    = chunk_size
+        @pipeline_size = pipeline_size
+
+        @receiving = {}
+      end
+
+      def receive_file(file_size, checksum)
+        FileService::ReceivingFile.new(
+          file_size, checksum, chunk_size, pipeline_size
+        ).tap do |info|
+          @receiving[info.file_uuid] = info
+        end
+      end
+
+      def receive_message(from_identity, message)
+        if current_file = @receiving[message.to]
+          current_file.sender_uuid = from_identity
+        else
+          return
+        end
+
+        case message.body[0]
+        when "START"
+          Pantry.logger.debug("[Receive File] Received START message #{message.inspect}")
+          fill_the_pipeline(current_file, message)
+        when "CHUNK"
+          Pantry.logger.debug("[Receive File] Received CHUNK message #{message.metadata}")
+          process_chunk(current_file, message)
+        end
+      end
+
+      protected
+
+      def fill_the_pipeline(current_file, message)
+        current_file.chunks_to_fetch do |offset, size|
+          Pantry.logger.debug("[Receive File] Fetching #{offset} x #{size} for #{current_file.file_uuid}")
+          send_message(current_file, "FETCH", offset, size)
+        end
+      end
+
+      def process_chunk(current_file, message)
+        chunk_offset = message[:chunk_offset]
+        chunk_size   = message[:chunk_size]
+        chunk_data   = message.body[1]
+
+        current_file.write_chunk(chunk_offset, chunk_size, chunk_data)
+
+        if current_file.complete?
+          finalize_file(current_file)
+        else
+          fill_the_pipeline(current_file, message)
+        end
+      end
+
+      def finalize_file(current_file)
+        if current_file.valid?
+          Pantry.logger.debug("[Receive File] File #{current_file.file_uuid} finished")
+          send_message(current_file, "FINISH")
+        else
+          Pantry.logger.debug("[Receive File] File #{current_file.file_uuid} did not upload successfully")
+          current_file.remove
+          send_message(current_file, "ERROR", "Checksum did not match the uploaded file")
+        end
+
+        current_file.finished!
+        @receiving.delete(current_file.file_uuid)
+      end
+
+      def send_message(current_file, *body)
+        message    = Pantry::Message.new
+        message.to = current_file.file_uuid
+
+        body.each {|part| message << part }
+
+        @service.send_message(current_file.sender_uuid, message)
+      end
+
+    end
+
+  end
+end

data/lib/pantry/communication/file_service/send_file.rb

@@ -0,0 +1,74 @@
+module Pantry
+  module Communication
+
+    # Chunk file sending tool that implements the protocol as described here
+    #  http://zguide.zeromq.org/page:all#Transferring-Files
+    #
+    # As this actor receives chunk requests from the Receiver, it reads that chunk
+    # from the given file and sends it along.
+    class FileService::SendFile
+      include Celluloid
+
+      def initialize(service)
+        @service = service
+        @sending = {}
+      end
+
+      def send_file(file_path, receiver_uuid, file_uuid)
+        sender_info = FileService::SendingFile.new(file_path, receiver_uuid, file_uuid)
+
+        @sending[file_uuid] = sender_info
+        send_message(sender_info, "START")
+
+        sender_info
+      end
+
+      def receive_message(from_identity, message)
+        current_file_info = @sending[message.to]
+        return unless current_file_info
+
+        case message.body[0]
+        when "FETCH"
+          Pantry.logger.debug("[Send File] FETCH requested #{message.inspect}")
+          fetch_and_return_chunk(current_file_info, message)
+        when "FINISH"
+          Pantry.logger.debug("[Send File] FINISHED cleaning up for #{message.inspect}")
+          clean_up(current_file_info, message)
+        when "ERROR"
+          Pantry.logger.debug("[Send File] ERROR #{message.inspect}")
+        end
+      end
+
+      protected
+
+      def fetch_and_return_chunk(current_file, message)
+        chunk_offset = message.body[1].to_i
+        chunk_size   = message.body[2].to_i
+
+        chunk = current_file.read(chunk_offset, chunk_size)
+
+        send_message(current_file, ["CHUNK", chunk], chunk_offset: chunk_offset, chunk_size: chunk_size)
+      end
+
+      def clean_up(current_file, message)
+        current_file.finished!
+        @sending.delete(message.to)
+      end
+
+      def send_message(sender_info, body, metadata = {})
+        message    = Pantry::Message.new
+        message.to = sender_info.file_uuid
+
+        [body].flatten.each {|part| message << part }
+
+        metadata.each do |key, value|
+          message[key] = value
+        end
+
+        @service.send_message(sender_info.receiver_uuid, message)
+      end
+
+    end
+
+  end
+end

data/lib/pantry/communication/publish_socket.rb

@@ -0,0 +1,20 @@
+module Pantry
+  module Communication
+
+    # The PublishSocket handles the Publish side of Pub/Sub using
+    # a 0MQ PUB socket. Messages can be published to all listening clients
+    # or can be filtered to certain clients using a ClientFilter.
+    # See SubscribeSocket for the receiving end.
+    class PublishSocket < WritingSocket
+
+      def build_socket
+        Celluloid::ZMQ::PubSocket.new
+      end
+
+      def open_socket(socket)
+        socket.bind("tcp://#{host}:#{port}")
+      end
+
+    end
+  end
+end

data/lib/pantry/communication/reading_socket.rb

@@ -0,0 +1,89 @@
+module Pantry
+  module Communication
+
+    # Base class of all sockets that read messages from ZMQ.
+    # Not meant for direct use, please use one of the subclasses for specific
+    # functionality.
+    class ReadingSocket
+      include Celluloid::ZMQ
+      finalizer :shutdown
+
+      attr_reader :host, :port
+
+      def initialize(host, port, security)
+        @host     = host
+        @port     = port
+        @listener = nil
+        @security = security
+      end
+
+      def add_listener(listener)
+        @listener = listener
+      end
+
+      def open
+        @socket = build_socket
+        Communication.configure_socket(@socket)
+        @security.configure_socket(@socket)
+        open_socket(@socket)
+
+        @running = true
+        self.async.process_messages
+      end
+
+      def build_socket
+        raise "Implement the socket setup."
+      end
+
+      def open_socket(socket)
+        raise "Connect / Bind the socket built in #build_socket"
+      end
+
+      def shutdown
+        @running = false
+      end
+
+      # Some ZMQ socket types include the source as the first packet of a message.
+      # We need to know if the socket in question does this so we can properly
+      # build the Message coming in.
+      def has_source_header?
+        false
+      end
+
+      protected
+
+      def process_messages
+        while @running
+          process_next_message
+        end
+
+        @socket.close
+      end
+
+      def process_next_message
+        next_message = []
+
+        # Drop the ZMQ given source packet, it's extraneous for our purposes
+        if has_source_header?
+          @socket.read
+        end
+
+        next_message << @socket.read
+
+        while @socket.more_parts?
+          next_message << @socket.read
+        end
+
+        async.handle_message(
+          SerializeMessage.from_zeromq(next_message)
+        )
+      end
+
+      def handle_message(message)
+        @listener.handle_message(message) if @listener
+      end
+
+    end
+
+  end
+end

data/lib/pantry/communication/receive_socket.rb

@@ -0,0 +1,23 @@
+module Pantry
+  module Communication
+
+    # The ReceiveSocket receives communication from Clients via the
+    # Dealer / Router socket pair. This class is the Server's Router side.
+    class ReceiveSocket < ReadingSocket
+
+      def build_socket
+        Celluloid::ZMQ::RouterSocket.new
+      end
+
+      def open_socket(socket)
+        socket.bind("tcp://#{host}:#{port}")
+      end
+
+      def has_source_header?
+        true
+      end
+
+    end
+
+  end
+end