pantry 0.0.0 → 0.1.0

data/lib/pantry/cli.rb

@@ -0,0 +1,154 @@
+module Pantry
+
+  # Pantry's Command Line Interface.
+  class CLI < Client
+
+    def initialize(command_line, **args)
+      @command_line = Pantry::CommandLine.new(command_line)
+
+      args[:identity] ||= ENV["USER"]
+      super(**args)
+    end
+
+    def run
+      prepare_local_pantry_root
+
+      options, arguments = @command_line.parse!
+      if options && process_global_options(options)
+        super
+        perform(options, arguments)
+      end
+
+      terminate
+    end
+
+    def prepare_local_pantry_root
+      if Pantry.config.root_dir.nil?
+        # TODO Find a .pantry up the chain vs building one
+        Pantry.config.root_dir = File.join(Dir.pwd, ".pantry")
+        FileUtils.mkdir_p(Pantry.root)
+      end
+    end
+
+    def process_global_options(options)
+      if options["verbose"]
+        Pantry.config.log_level = :info
+        Pantry.config.refresh
+      end
+
+      if options["debug"]
+        Pantry.config.log_level = :debug
+        Pantry.config.refresh
+      end
+
+      if server_host = options["host"]
+        Pantry.config.server_host = server_host
+      end
+
+      if curve_key_file = options["curve-key-file"]
+        Pantry.config.security = "curve"
+        copy_keys_file_into_pantry_root(curve_key_file)
+      end
+
+      if options["version"]
+        Pantry.ui.say Pantry::VERSION
+        terminate
+        return false
+      end
+
+      true
+    end
+
+    # Given the parsed options and the arguments left over,
+    # figure out what Command was requested, build up the appropriate structures
+    # and start the communication process.
+    def perform(options, arguments)
+      command_info, command_options = @command_line.triggered_command
+
+      if command_info
+        client_filter = Pantry::Communication::ClientFilter.new(
+          application: options['application'],
+          environment: options['environment'],
+          roles:       options['roles']
+        )
+
+        command = command_info[:class].new(*arguments)
+        command_options = command_options.merge(options)
+
+        request(client_filter, command, command_options)
+      else
+        $stderr.puts "I don't know the #{arguments.first} command"
+      end
+    end
+
+    # Fire off the requested Command.
+    def request(filter, command, options)
+      @command = command
+      @command.client = self
+
+      # We don't use send_request here because we don't want to deal with the
+      # wait-list future system. This lets command objects handle responses
+      # as they come back to the CLI as the command sees fit.
+      # If the command isn't meant directly for the Server, the Server will always
+      # respond first with the list of clients who will be executing the command
+      # and responding with the results. See Pantry::Commands::Echo for an example of how
+      # to work with this flow.
+      begin
+        message = @command.prepare_message(options)
+        message.to = filter.stream
+        message.requires_response!
+
+        send_message(message)
+
+        @command.wait_for_finish
+
+        copy_full_keys_back_to_curve_key_file
+      rescue Exception => ex
+        Pantry.ui.say("Error: #{ex.message}")
+        Pantry.logger.debug(ex.backtrace.join("\n"))
+      end
+    end
+
+    # All messages received by this client are assumed to be responses
+    # from previous commands.
+    def receive_message(message)
+      if @command
+        @command.receive_response(message)
+      end
+    end
+
+    protected
+
+    #
+    # For the sake of being a Client, the file mentioned in --curve-key-file is
+    # copied into security/curve/client_keys.yml so that we don't have to do anything
+    # special to turn on Curve encryption for the Client. We do this for every run to ensure
+    # that if the keys or options change, the user isn't confused when the client is unable to
+    # connect or auth.
+    #
+    # To facilitate the first-time-connect situation, where a new Pantry Server has been spun up
+    # and the only known key is the server's public key, this will also copy the keys file back
+    # up to the file named in --curve-key-file because it will fill that file with a set of
+    # generated public and private keys.
+    #
+
+    def copy_keys_file_into_pantry_root(curve_key_file)
+      @curve_key_file = curve_key_file
+      FileUtils.mkdir_p(Pantry.root.join("security", "curve"))
+      FileUtils.cp(
+        Pantry.root.join(@curve_key_file),
+        Pantry.root.join("security", "curve", "client_keys.yml")
+      )
+    end
+
+    def copy_full_keys_back_to_curve_key_file
+      return unless @curve_key_file
+      FileUtils.cp(
+        Pantry.root.join("security", "curve", "client_keys.yml"),
+        Pantry.root.join(@curve_key_file)
+      )
+    end
+
+  end
+
+end

data/lib/pantry/client.rb

@@ -0,0 +1,131 @@
+module Pantry
+
+  # The Pantry Client.
+  # The Client runs on any server that needs provisioning,
+  # and communicates to the Server through various channels. Clients can
+  # be further configured to manage an application, for a given environment,
+  # and across any number of roles.
+  class Client
+    extend Forwardable
+    include Celluloid
+    finalizer :shutdown
+
+    # See Pantry::ClientInfo
+    def_delegators :@info, :identity, :application, :environment, :roles, :filter
+
+    # For testing / debugging purposes, keep hold of the last message this client received
+    attr_reader :last_received_message
+
+    def initialize(application: nil, environment: nil, roles: [], identity: nil, network_stack_class: Communication::Client)
+      @info      = Pantry::ClientInfo.new(
+        application: application,
+        environment: environment,
+        roles:       roles       || [],
+        identity:    identity    || current_hostname
+      )
+
+      @commands   = CommandHandler.new(self, Pantry.client_commands)
+      @networking = network_stack_class.new_link(self)
+    end
+
+    # Start up the Client.
+    # This sets up the appropriate communication channels to the
+    # server, sends a registration message so the Server knows who
+    # just connected, and then waits for information to come.
+    def run
+      Pantry.set_proc_title("pantry client #{Pantry::VERSION} :: #{identity}")
+      @networking.run
+      send_registration_message
+      Pantry.logger.info("[#{identity}] Client registered and waiting for commands")
+    end
+
+    def shutdown
+      Pantry.logger.info("[#{identity}] Client Shutting down")
+      @registration_timer.cancel if @registration_timer
+    end
+
+    # Callback from Networking when a message is received
+    def receive_message(message)
+      Pantry.logger.debug("[#{identity}] Received message #{message.inspect}")
+
+      if message_meant_for_us?(message)
+        @last_received_message = message
+        results = @commands.process(message)
+
+        if message.requires_response?
+          Pantry.logger.debug("[#{identity}] Responding with #{results.inspect}")
+          send_results_back_to_requester(message, results)
+        end
+      else
+        Pantry.logger.debug("[#{identity}] Message discarded, not for us")
+      end
+    end
+
+    # Send a Pantry::Message directly to its intended recipient.
+    # For a Client this is almost always the Server.
+    def send_message(message)
+      @networking.send_message(message)
+    end
+
+    # Send a Pantry::Message but mark it as requiring a response.
+    # This will set up and return a Celluloid::Future that will contain the
+    # response once it is available.
+    def send_request(message)
+      message.requires_response!
+
+      Pantry.logger.debug("[#{identity}] Sending request #{message.inspect}")
+
+      @networking.send_request(message)
+    end
+
+    # See Pantry::Server#receive_file
+    def receive_file(file_size, file_checksum)
+      @networking.receive_file(file_size, file_checksum)
+    end
+
+    # See Pantry::Server#send_file
+    def send_file(file_path, receiver_uuid, file_uuid)
+      @networking.send_file(file_path, receiver_uuid, file_uuid)
+    end
+
+    protected
+
+    def current_hostname
+      Socket.gethostname
+    end
+
+    def send_registration_message
+      @networking.send_message(
+        Pantry::Commands::RegisterClient.new(self).to_message
+      )
+      @registration_timer =
+        after(Pantry.config.client_heartbeat_interval) { send_registration_message }
+    end
+
+    def send_results_back_to_requester(message, results)
+      response_message = message.build_response
+
+      [results].flatten(1).each do |entry|
+        response_message << entry
+      end
+
+      @networking.send_message(response_message)
+    end
+
+    # ZeroMQ's Pub/Sub topic matching is too simplistic to catch all the cases we
+    # need to handle. Given that if *any* topic matches the incoming message, we get
+    # the message even if it wasn't exactly meant for us. For example, if this client
+    # subscribes to the following topics:
+    #
+    #   * pantry
+    #   * pantry.test
+    #   * pantry.test.app
+    #
+    # This client will receive messages sent to "pantry.test.web" because "pantry" and
+    # "pantry.test" both match (string start_with? check) the message. Thus, we add our
+    # own handling to the message check as a protective stop gap.
+    def message_meant_for_us?(message)
+      filter.matches?(message.to)
+    end
+  end
+end

data/lib/pantry/client_info.rb

@@ -0,0 +1,34 @@
+module Pantry
+
+  # Simple class to keep track of a given client's identifying information
+  class ClientInfo
+    attr_reader :application
+
+    attr_reader :environment
+
+    attr_reader :roles
+
+    # The above gets packaged into a ClientFilter for use elsewhere
+    attr_reader :filter
+
+    # This client's current identity. By default a client's identity is it's `hostname`,
+    # but a specific one can be given. These identities should be unique across the set
+    # of clients connecting to a single Pantry Server, behavior of multiple clients with
+    # the same identity is currently undefined.
+    attr_reader :identity
+
+    def initialize(application: nil, environment: nil, roles: [], identity: nil)
+      @application = application
+      @environment = environment
+      @roles       = roles
+      @identity    = identity
+
+      @filter = Pantry::Communication::ClientFilter.new(
+        application: @application,
+        environment: @environment,
+        roles:       @roles,
+        identity:    @identity
+      )
+    end
+  end
+end

data/lib/pantry/client_registry.rb

@@ -0,0 +1,104 @@
+module Pantry
+
+  # The ClientRegistry keeps track of clients who've checked in and supports
+  # various querying requests against the list of known clients.
+  class ClientRegistry
+
+    def initialize
+      clear!
+    end
+
+    # Return all known clients
+    def all
+      @clients.map {|identity, record| record.client }
+    end
+
+    # Clear out the registry entirely
+    def clear!
+      @clients = Hash.new {|hash, key| hash[key] = ClientRecord.new }
+    end
+
+    # Check in a client
+    def check_in(client)
+      @clients[client.identity].check_in(client)
+    end
+
+    # Has the given client checked in?
+    def include?(client)
+      @clients[client.identity].checked_in?
+    end
+
+    # Find info for Client that matches the given identity
+    def find(identity)
+      if found = @clients[identity]
+        found.client
+      else
+        nil
+      end
+    end
+
+    # Find and return all clients who will receive messages
+    # on the given stream or ClientFilter.
+    #
+    # If this method is given a block, the block will be processed as
+    # a #map of the list of clients and records. Block expected to be
+    # of the form:
+    #
+    #   all_matching(filter) do |client, record|
+    #     ...
+    #   end
+    #
+    # The `record` contains internal knowledge of the Client's activity.
+    # See ClientRecord for what's contained.
+    def all_matching(stream_or_filter)
+      found_client_records =
+        case stream_or_filter
+        when String
+          select_records_matching do |record|
+            record.client.filter.matches?(stream_or_filter)
+          end
+        else
+          select_records_matching do |record|
+            stream_or_filter.includes?(record.client.filter)
+          end
+        end
+
+      if block_given?
+        found_client_records.map do |record|
+          yield(record.client, record)
+        end
+      else
+        found_client_records.map(&:client)
+      end
+    end
+
+    protected
+
+    def select_records_matching
+      selected_records = @clients.clone.select do |identity, record|
+        yield(record)
+      end
+
+      selected_records.values
+    end
+
+    class ClientRecord
+      attr_reader :client, :last_checked_in_at
+
+      def initialize
+        @client = nil
+        @last_checked_in_at = nil
+      end
+
+      def check_in(client)
+        @client = client
+        @last_checked_in_at = Time.now
+      end
+
+      def checked_in?
+        !@last_checked_in_at.nil?
+      end
+    end
+
+  end
+end

data/lib/pantry/command.rb

@@ -0,0 +1,194 @@
+module Pantry
+
+  # Commands are where the task-specific functionality is implemented, the
+  # core of how Pantry works. All Commands are required to implement the #perform method,
+  # which receives the Pantry::Message requesting the Command.
+  #
+  # All commands must be registered with Pantry before they will be available for execution.
+  # Use Pantry.add_client_command or Pantry.add_server_command to register the Command.
+  class Command
+
+    class << self
+      # Expose this Command to the CLI and configure the options and information
+      # that this Command needs from the CLI to function.
+      #
+      # See OptParsePlus for documentation
+      def command(name, &block)
+        @command_name   = name
+        @command_config = block
+      end
+      attr_reader :command_name
+      attr_reader :command_config
+    end
+
+    # Initialize this Command
+    # Due to the multiple ways a Command is instantiated (via the CLI and from the Network stack)
+    # any Command initializer must support being called with zero parameters if it expects some.
+    def initialize(*args)
+    end
+
+    # Set up the Message that needs to be created to send this Command
+    # to the server to be processed. Used by the CLI. This method is given
+    # the ClientFilter of which clients should respond to this message (if any) and
+    # the extra arguments given on the command line.
+    #
+    # If work needs to be done prior to the network communication for CLI use,
+    # override method to take care of that logic.
+    #
+    # The message returned here is then passed through the network to the appropriate
+    # recipients (Clients, Server, or both) and used to trigger #perform on said
+    # recipient.
+    def prepare_message(options)
+      to_message
+    end
+
+    # Run whatever this command needs to do.
+    # All Command subclasses must implement this method.
+    # If the message triggering this Command expects a response, the return
+    # value of this method will be that response.
+    def perform(message)
+    end
+
+    # When a message comes back from the server as a response to or because of
+    # this command's #perform, the command object on the CLI will receive that
+    # message here. This method will dispatch to either #receive_server_response
+    # or #receive_client_response depending on the type of Command run.
+    # In most cases, Commands should override the server/client specific receivers.
+    # Only override this method to fully customize Message response handling.
+    def receive_response(response)
+      @response_tracker ||= TrackResponses.new
+      @response_tracker.new_response(response)
+
+      if @response_tracker.from_server?
+        receive_server_response(response)
+        finished
+      elsif @response_tracker.from_client?
+        receive_client_response(response)
+        finished if @response_tracker.all_response_received?
+      end
+    end
+
+    # Handle a response from a Server Command. Override this for specific handling
+    # of Server Command responses.
+    def receive_server_response(response)
+      Pantry.ui.say("Server response:")
+      Pantry.ui.say(response.body.inspect)
+    end
+
+    # Handle a response from a Client Command. This will be called for each Client
+    # executing and responding to the requested Command.
+    def receive_client_response(response)
+      Pantry.ui.say("Response from #{response.from}:")
+      Pantry.ui.say(response.body.inspect)
+    end
+
+    # Send a request out, returning the Future which will eventually
+    # contain the response Message
+    def send_request(message)
+      @server_or_client.send_request(message)
+    end
+
+    # Send a request out and wait for the response. Will return the response
+    # once it is received.
+    #
+    # This is a blocking call.
+    def send_request!(message)
+      send_request(message).value
+    end
+
+    # Create a new Message from the information in the current Command
+    def to_message
+      Pantry::Message.new(self.class.message_type)
+    end
+
+    # Blocking call that returns when the command has completed
+    # Can be given a timeout (in seconds) to wait for a response
+    def wait_for_finish(timeout = nil)
+      completion_future.value(timeout)
+    end
+
+    # Notify all listeners that this command has completed its tasks
+    def finished
+      completion_future.signal(OpenStruct.new(:value => nil))
+    end
+
+    # Is this command finished?
+    def finished?
+      completion_future.ready?
+    end
+
+    def completion_future
+      @completion_future ||= Celluloid::Future.new
+    end
+    protected :completion_future
+
+    # The Type of this command, used to differentiate Messages.
+    # Defaults to the full scope of the name, though with the special
+    # case of removing any "Pantry" related scoping such as Pantry::
+    # and Pantry::Commands::
+    #
+    # This value must be unique across the system or the messages will not
+    # be processed reliably.
+    #
+    # Override this for a custom name.
+    def self.message_type
+      self.name.gsub(/Pantry::Commands::/, '').gsub(/Pantry::/, '')
+    end
+
+    # Set a link back to the Server or Client that's handling
+    # this command. This will be set by the CommandHandler before calling
+    # #perform.
+    def server_or_client=(server_or_client)
+      @server_or_client = server_or_client
+    end
+    alias client= server_or_client=
+    alias server= server_or_client=
+
+    # Get the server or client object handling this command
+    def server_or_client
+      @server_or_client
+    end
+    alias server server_or_client
+    alias client server_or_client
+
+    protected
+
+    # Internal state tracking of server and client responses.
+    # When a Client Command is triggered, the Server first responses with a message
+    # containing the list of Clients who will execute the Command and respond.
+    # Then we need to keep track of all the Clients who have responded so we know
+    # when the command has fully finished across all Clients.
+    class TrackResponses
+      def initialize
+        @expected_clients      = []
+        @received_from_clients = []
+      end
+
+      def new_response(response)
+        @latest_response = response
+
+        if response[:client_response_list]
+          @expected_clients = response.body
+          count = @expected_clients.length
+          Pantry.ui.say("Expecting response from #{count} client#{count == 1 ? "" : "s"}...")
+        elsif from_client?
+          @received_from_clients << response
+        end
+      end
+
+      def from_server?
+        @latest_response.from_server? && !@latest_response[:client_response_list]
+      end
+
+      def from_client?
+        !@latest_response.from_server?
+      end
+
+      def all_response_received?
+        !@expected_clients.empty? && @expected_clients.length == @received_from_clients.length
+      end
+    end
+
+  end
+
+end