pantry 0.0.0 → 0.1.0

data/lib/pantry/commands/sync_directory.rb

@@ -0,0 +1,50 @@
+module Pantry
+  module Commands
+
+    # Base class for any Command that needs to sync a set of files in a directory
+    # from the Server down to the Client.
+    #
+    # Subclasses need to define where on the server the files live and where on
+    # the client the files will be written to. Both #server_directory and #client_directory
+    # are executed on the Client so #client is accessible for added information.
+    #
+    # This command expects simple directories with a small number of files that are
+    # themselves small in size, as this command reads every file into memory and sends
+    # that raw content back to the Client. If there are more substantial files to transfer
+    # use #send_file and #receive_file instead.
+    class SyncDirectory < Pantry::Command
+
+      def server_directory(local_root)
+        raise "Specify the read directory on the server"
+      end
+
+      def client_directory(local_root)
+        raise "Specify the write directory on the client"
+      end
+
+      def perform(message)
+        dir_contents = send_request!(
+          Pantry::Commands::DownloadDirectory.new(
+            server_directory(Pathname.new(""))
+          ).to_message
+        )
+
+        write_to = client_directory(Pantry.root)
+        FileUtils.mkdir_p(write_to)
+
+        dir_contents.body.each do |(file_name, file_contents)|
+          file_path = write_to.join(file_name).cleanpath
+          FileUtils.mkdir_p(File.dirname(file_path))
+
+          File.open(file_path, "w+") do |file|
+            file.write(file_contents)
+          end
+        end
+
+        true
+      end
+
+    end
+
+  end
+end

data/lib/pantry/commands/update_application.rb

@@ -0,0 +1,45 @@
+module Pantry
+  module Commands
+
+    # Upload and save new configuration for an Application
+    #
+    # See EditApplication for more information
+    class UpdateApplication < Pantry::Command
+
+      def initialize(application_name = nil, config_body = nil)
+        @application_name = application_name
+        @config_body      = config_body
+      end
+
+      def to_message
+        super.tap do |msg|
+          msg << @application_name
+          msg << @config_body
+        end
+      end
+
+      def perform(message)
+        application_name = message.body[0]
+        config_body      = message.body[1]
+
+        app_config_file = Pantry.root.join("applications", application_name, "config.yml")
+        FileUtils.mkdir_p(File.dirname(app_config_file))
+
+        begin
+          Psych.parse(config_body, "config.yml")
+        rescue => ex
+          # Invalid YAML, don't save!
+          return [false, ex.message]
+        end
+
+        File.open(app_config_file, "w+") do |file|
+          file.write(config_body)
+        end
+
+        true
+      end
+
+    end
+
+  end
+end

data/lib/pantry/commands/upload_file.rb

@@ -0,0 +1,68 @@
+module Pantry
+  module Commands
+
+    # Base class for any command that needs to upload a single file where
+    # that file is small enough for its contents to be passed around in plain messages.
+    # For larger files that shouldn't be pulled entirely into memory, please use
+    # #send_file and #receive_file instead.
+    #
+    # This class is not used directly. Subclass to define the CLI command pattern
+    # and the directory where the uploaded file will end up.
+    class UploadFile < Pantry::Command
+
+      attr_reader :file_to_upload
+
+      def initialize(file_to_upload = nil)
+        @file_to_upload = file_to_upload
+      end
+
+      # Specify the directory this file should be written to
+      # When applicable, +application+ is the application that should know about this file
+      def upload_directory(application)
+        raise "Must implement #upload_directory in subclass"
+      end
+
+      # Specify any required options for this Command by long-name
+      # For example, to require the base APPLICATION option, return %i(application)
+      # Does not matter if the list is of strings or symbols.
+      def required_options
+        []
+      end
+
+      def prepare_message(options)
+        required_options.each do |required|
+          unless options[required]
+            raise Pantry::MissingOption, "Required option #{required} is missing"
+          end
+        end
+
+        super.tap do |message|
+          message << options
+          message << File.basename(@file_to_upload)
+          message << File.read(@file_to_upload)
+        end
+      end
+
+      def perform(message)
+        cmd_options = message.body[0]
+        file_name   = message.body[1]
+        file_body   = message.body[2]
+
+        upload_dir = upload_directory(cmd_options)
+
+        FileUtils.mkdir_p(upload_dir)
+        File.open(upload_dir.join(file_name), "w+") do |file|
+          file.write file_body
+        end
+
+        true
+      end
+
+      def receive_server_response(response)
+        # Say nothing. Finishing is enough
+      end
+
+    end
+
+  end
+end

data/lib/pantry/communication.rb

@@ -0,0 +1,20 @@
+module Pantry
+
+  # The Communication subsystem of Pantry is managed via 0MQ through the
+  # Celluloid::ZMQ library.
+  module Communication
+    Celluloid::ZMQ.init
+
+    # Configure a ZMQ socket with some common options
+    def self.configure_socket(socket)
+      # Ensure the socket doesn't spam us trying to reconnect
+      # after a disconnect or authentication failure
+      socket.set(::ZMQ::RECONNECT_IVL,     1_000)
+      socket.set(::ZMQ::RECONNECT_IVL_MAX, 30_000)
+
+      # Drop all messages on shutdown
+      socket.linger = 0
+    end
+  end
+
+end

data/lib/pantry/communication/client.rb

@@ -0,0 +1,75 @@
+module Pantry
+  module Communication
+
+    # The communication layer of a Pantry::Client
+    # This class manages all of the ZeroMQ sockets and underlying
+    # communication systems, handling the sending and receiving of messages.
+    class Client
+      include Celluloid
+
+      def initialize(listener)
+        @listener = listener
+        @response_wait_list = Communication::WaitList.new
+      end
+
+      # Start up the networking layer, opening up sockets and getting
+      # ready for communication.
+      def run
+        @security = Communication::Security.new_client
+
+        @subscribe_socket = Communication::SubscribeSocket.new_link(
+          Pantry.config.server_host,
+          Pantry.config.pub_sub_port,
+          @security
+        )
+        @subscribe_socket.add_listener(self)
+        @subscribe_socket.filter_on(@listener.filter)
+        @subscribe_socket.open
+
+        @send_socket = Communication::SendSocket.new_link(
+          Pantry.config.server_host,
+          Pantry.config.receive_port,
+          @security
+        )
+        @send_socket.open
+
+        @file_service = Communication::FileService.new_link(
+          Pantry.config.server_host,
+          Pantry.config.file_service_port,
+          @security
+        )
+        @file_service.start_client
+      end
+
+      # Callback from the SubscribeSocket when a message is received.
+      def handle_message(message)
+        if @response_wait_list.waiting_for?(message)
+          @response_wait_list.received(message)
+        else
+          @listener.receive_message(message)
+        end
+      end
+
+      def send_request(message)
+        @response_wait_list.wait_for(message).tap do
+          send_message(message)
+        end
+      end
+
+      def send_message(message)
+        message.from = @listener
+        @send_socket.send_message(message)
+      end
+
+      def receive_file(file_size, file_checksum)
+        @file_service.receive_file(file_size, file_checksum)
+      end
+
+      def send_file(file_path, receiver_uuid, file_uuid)
+        @file_service.send_file(file_path, receiver_uuid, file_uuid)
+      end
+
+    end
+
+  end
+end

data/lib/pantry/communication/client_filter.rb

@@ -0,0 +1,117 @@
+module Pantry
+  module Communication
+
+    # ClientFilter handles and manages building filters that map configuration values
+    # to 0MQ stream names. A message stream is a period-delimited string that works
+    # with 0MQ's Subscription prefix matcher, allowing Clients to choose which messages
+    # they want to receive. Streams are built to enable tiered delivery capability.
+    #
+    # For example, a Client with the application 'pantry' and roles 'app' and 'db' will
+    # subscribe to the following streams:
+    #
+    #   pantry
+    #   pantry.app
+    #   pantry.db
+    #
+    # Similarly to differentiate between environments, the environment is added inbetween
+    # the application and the role:
+    #
+    #   pantry
+    #   pantry.production
+    #   pantry.production.app
+    #   pantry.production.db
+    #
+    # This class is also used to when sending messages, to choose which specific stream
+    # (e.g. the deepest buildable) to send a given message down.
+    #
+    # A client identity token can also be given via +identity+. If identity is provided
+    # then that stream will be chosen above all others. Use this to send a message to
+    # specific clients.
+    class ClientFilter
+
+      attr_reader :application, :environment, :roles, :identity
+
+      def initialize(application: nil, environment: nil, roles: [], identity: nil)
+        @application = application
+        @environment = environment
+        @roles       = roles || []
+        @identity    = identity
+      end
+
+      # List out all communication streams this ClientFilter is configured to know about.
+      def streams
+        list = []
+        base_stream = []
+
+        if @identity
+          list << @identity
+        end
+
+        if @application
+          list << @application
+          base_stream = [@application]
+        end
+
+        if @environment
+          list << [base_stream, @environment].flatten.compact.join(".")
+        end
+
+        @roles.each do |role|
+          list << [base_stream, @environment, role].flatten.compact.join(".")
+        end
+
+        list = list.flatten.compact
+        list.empty? ? [""] : list
+      end
+
+      # Return the most specific stream that matches this ClientFilter.
+      # +identity+ is chosen above all others.
+      def stream
+        if @identity
+          @identity
+        else
+          [@application, @environment, @roles.first].compact.join(".")
+        end
+      end
+
+      def ==(other)
+        return false unless other
+        return false unless other.is_a?(ClientFilter)
+
+        self.application   == other.application &&
+          self.environment == other.environment &&
+          self.roles       == other.roles &&
+          self.identity    == other.identity
+      end
+
+      # Will this filter match on the given stream?
+      def matches?(test_stream)
+        self.streams.any? do |stream|
+          stream.start_with?(test_stream)
+        end
+      end
+
+      # A filter includes another filter if the other filter matches.
+      # This does not look at identities.
+      def includes?(filter)
+        return true if self == filter
+        return true if streams == [""]
+
+        my_stream =    Set.new(streams)
+        other_stream = Set.new(filter.streams)
+
+        my_stream.subset?(other_stream)
+      end
+
+      def to_hash
+        {
+          application: @application,
+          environment: @environment,
+          roles:       @roles,
+          identity:    @identity
+        }
+      end
+
+    end
+  end
+end

data/lib/pantry/communication/file_service.rb

@@ -0,0 +1,125 @@
+module Pantry
+  module Communication
+
+    # FileService manages the sending and receiving of files that are too big
+    # to cleanly send as a plain ZeroMQ message.
+    # Every Client and Server has its own FileService handler which can manage
+    # both sending and receiving files from each other.
+    #
+    # Setting up a file transfer processes backwards from what may be expected.
+    # As the Receiver actually requests chunks from the Sender, a protocol that's
+    # heavily influenced by http://zguide.zeromq.org/page:all#Transferring-Files,
+    # a Receiver must be initiated first on the receiving end, which will then pass
+    # back the appropriate information (receiver_uuid and file upload UUID) a
+    # Sender needs to start up and run.
+    #
+    # From this the two parts complete the process automatically. A Receiver writes the
+    # data it receives in a tempfile, and must be configured with a completion block
+    # to move the uploaded file to its final location.
+    #
+    # To ensure this object has as little special-casing code as possible, the communication
+    # takes place in a ZeroMQ ROUTER <-> ROUTER topology.
+    class FileService
+      include Celluloid::ZMQ
+      finalizer :shutdown
+
+      attr_reader :identity
+
+      def initialize(server_host, port, security)
+        @host = server_host
+        @port = port
+
+        @socket = Celluloid::ZMQ::RouterSocket.new
+        @socket.set(::ZMQ::ROUTER_MANDATORY, 1)
+        @socket.identity = @identity = SecureRandom.uuid
+        Communication.configure_socket(@socket)
+
+        @security = security
+        @security.configure_socket(@socket)
+
+        @receiver = FileService::ReceiveFile.new_link(self)
+        @sender   = FileService::SendFile.new_link(self)
+      end
+
+      def secure_with(security_handler)
+        @security = security_handler
+      end
+
+      def start_server
+        @socket.bind("tcp://#{@host}:#{@port}")
+        run
+      end
+
+      def start_client
+        @socket.connect("tcp://#{@host}:#{@port}")
+        run
+      end
+
+      def shutdown
+        @running = false
+      end
+
+      def run
+        @running = true
+        self.async.process_messages
+      end
+
+      # Inform the service that it will soon be receiving a file of the given
+      # size and checksum. Returns a UploadInfo struct with the information for
+      # the Sender.
+      def receive_file(size, checksum)
+        Pantry.logger.debug("[FileService] Receiving file of size #{size} and checksum #{checksum}")
+        @receiver.receive_file(size, checksum).tap do |info|
+          info.receiver_uuid = @socket.identity
+        end
+      end
+
+      # Inform the service that we want to start sending a file up to the receiver
+      # who's listening on the given UUID.
+      def send_file(file_path, receiver_uuid, file_uuid)
+        Pantry.logger.debug("[FileService] Sending file #{file_path} to #{receiver_uuid}")
+        @sender.send_file(file_path, receiver_uuid, file_uuid)
+      end
+
+      def send_message(identity, message)
+        @socket.write(
+          [
+            identity,
+            SerializeMessage.to_zeromq(message)
+          ].flatten
+        )
+      end
+
+      def receive_message(from_identity, message)
+        @sender.async.receive_message(from_identity, message)
+        @receiver.async.receive_message(from_identity, message)
+      end
+
+      protected
+
+      def process_messages
+        while @running
+          process_next_message
+        end
+
+        @socket.close
+      end
+
+      def process_next_message
+        next_message = []
+
+        from_identity = @socket.read
+
+        while @socket.more_parts?
+          next_message << @socket.read
+        end
+
+        async.receive_message(
+          from_identity, SerializeMessage.from_zeromq(next_message)
+        )
+      end
+
+    end
+
+  end
+end