chef-expander 0.10.0.beta.0

data/lib/chef/expander/configuration.rb

@@ -0,0 +1,307 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'pp'
+require 'optparse'
+require 'singleton'
+
+require 'chef/expander/flattener'
+require 'chef/expander/loggable'
+require 'chef/expander/version'
+
+module Chef
+  module Expander
+
+    def self.config
+      @config ||= Configuration::Base.new
+    end
+
+    def self.init_config(argv)
+      config.apply_defaults
+      remaining_opts_after_parse = Configuration::CLI.parse_options(argv)
+      # Need to be able to override the default config file location on the command line
+      config_file_to_use = Configuration::CLI.config.config_file || config.config_file
+      config.merge_config(Configuration::Base.from_chef_compat_config(config_file_to_use))
+      # But for all other config options, the CLI config should win over config file
+      config.merge_config(Configuration::CLI.config)
+      remaining_opts_after_parse
+    end
+
+    class ChefCompatibleConfig
+
+      attr_reader :config_hash
+
+      def initialize
+        @config_hash = {}
+      end
+
+      def load(file)
+        file = File.expand_path(file)
+        instance_eval(IO.read(file), file, 1) if File.readable?(file)
+      end
+
+      def method_missing(method_name, *args, &block)
+        if args.size == 1
+          @config_hash[method_name] = args.first
+        elsif args.empty?
+          @config_hash[method_name] or super
+        else
+          super
+        end
+      end
+
+    end
+
+    module Configuration
+
+      class InvalidConfiguration < StandardError
+      end
+
+      class Base
+
+        DEFAULT_PIDFILE = Object.new
+
+        include Loggable
+
+        def self.from_chef_compat_config(file)
+          config = ChefCompatibleConfig.new
+          config.load(file)
+          from_hash(config.config_hash)
+        end
+
+        def self.from_hash(config_hash)
+          config = new
+          config_hash.each do |setting, value|
+            setter = "#{setting}=".to_sym
+            if config.respond_to?(setter)
+              config.send(setter, value)
+            end
+          end
+          config
+        end
+
+        def self.configurables
+          @configurables ||= []
+        end
+
+        def self.validations
+          @validations ||= []
+        end
+
+        def self.defaults
+          @defaults ||= {}
+        end
+
+        def self.configurable(setting, default=nil, &validation)
+          attr_accessor(setting)
+          configurables << setting
+          defaults[setting] = default
+          validations << validation if block_given?
+
+          setting
+        end
+
+        configurable :config_file, "/etc/chef/solr.rb" do
+          unless (config_file && File.exist?(config_file) && File.readable?(config_file))
+            log.warn {"* " * 40}
+            log.warn {"Config file #{config_file} does not exist or cannot be read by user (#{Process.euid})"}
+            log.warn {"Default configuration settings will be used"}
+            log.warn {"* " * 40}
+          end
+        end
+
+        configurable :index do
+          invalid("You must specify this node's position in the ring as an integer") unless index.kind_of?(Integer)
+          invalid("The index cannot be larger than the cluster size (node-count)") unless (index <= node_count.to_i)
+        end
+
+        configurable :node_count do
+          invalid("You must specify the cluster size as an integer") unless node_count.kind_of?(Integer)
+          invalid("The cluster size (node-count) cannot be smaller than the index") unless node_count >= index.to_i
+        end
+
+        configurable :ps_tag, ""
+
+        configurable :solr_url, "http://localhost:8983"
+
+        configurable :amqp_host, '0.0.0.0'
+
+        configurable :amqp_port, 5672
+
+        configurable :amqp_user, 'chef'
+
+        configurable :amqp_pass, 'testing'
+
+        configurable :amqp_vhost, '/chef'
+
+        configurable :user, nil
+
+        configurable :group, nil
+
+        configurable :daemonize, false
+
+        alias :daemonize? :daemonize
+
+        configurable :pidfile, DEFAULT_PIDFILE
+
+        def pidfile
+          if @pidfile.equal?(DEFAULT_PIDFILE)
+            Process.euid == 0 ? '/var/run/chef-expander.pid' : '/tmp/chef-expander.pid'
+          else
+            @pidfile
+          end
+        end
+
+        configurable :log_level, :info
+
+        # override the setter for log_level to also actually set the level
+        def log_level=(level)
+          if level #don't accept nil for an answer
+            level = level.to_sym
+            Loggable::LOGGER.level = level
+            @log_level = log_level
+          end
+          level
+        end
+
+        configurable :log_location, STDOUT
+
+        # override the setter for log_location to re-init the logger
+        def log_location=(location)
+          Loggable::LOGGER.init(location) unless location.nil?
+        end
+
+        def initialize
+          reset!
+        end
+
+        def reset!(stdout=nil)
+          self.class.configurables.each do |setting|
+            send("#{setting}=".to_sym, nil)
+          end
+          @stdout = stdout || STDOUT
+        end
+
+        def apply_defaults
+          self.class.defaults.each do |setting, value|
+            self.send("#{setting}=".to_sym, value)
+          end
+        end
+
+        def merge_config(other)
+          self.class.configurables.each do |setting|
+            value = other.send(setting)
+            self.send("#{setting}=".to_sym, value) if value
+          end
+        end
+
+        def fail_if_invalid
+          validate!
+        rescue InvalidConfiguration => e
+          @stdout.puts("Invalid configuration: #{e.message}")
+          exit(1)
+        end
+
+        def invalid(message)
+          raise InvalidConfiguration, message
+        end
+
+        def validate!
+          self.class.validations.each do |validation_proc|
+            instance_eval(&validation_proc)
+          end
+        end
+
+        def vnode_numbers
+          vnodes_per_node = VNODES / node_count
+          lower_bound = (index - 1) * vnodes_per_node
+          upper_bound = lower_bound  + vnodes_per_node
+          upper_bound += VNODES % vnodes_per_node if index == node_count
+          (lower_bound...upper_bound).to_a
+        end
+
+        def amqp_config
+          {:host => amqp_host, :port => amqp_port, :user => amqp_user, :pass => amqp_pass, :vhost => amqp_vhost}
+        end
+
+      end
+
+      module CLI
+        @config = Configuration::Base.new
+
+        @option_parser = OptionParser.new do |o|
+          o.banner = "Usage: chef-expander [options]"
+
+          o.on('-c', '--config CONFIG_FILE', 'a configuration file to use') do |conf|
+            @config.config_file = File.expand_path(conf)
+          end
+
+          o.on('-i', '--index INDEX', 'the slot this node will occupy in the ring') do |i|
+            @config.index = i.to_i
+          end
+
+          o.on('-n', '--node-count NUMBER', 'the number of nodes in the ring') do |n|
+            @config.node_count = n.to_i
+          end
+
+          o.on('-l', '--log-level LOG_LEVEL', 'set the log level') do |l|
+            @config.log_level = l
+          end
+
+          o.on('-L', '--logfile LOG_LOCATION', 'Logfile to use') do |l|
+            @config.log_location = l
+          end
+
+          o.on('-d', '--daemonize', 'fork into the background') do
+            @config.daemonize = true
+          end
+
+          o.on('-P', '--pid PIDFILE') do |p|
+            @config.pidfile = p
+          end
+
+          o.on_tail('-h', '--help', 'show this message') do
+            puts "chef-expander #{Expander.version}"
+            puts ''
+            puts o
+            exit 1
+          end
+
+          o.on_tail('-v', '--version', 'show the version and exit') do
+            puts "chef-expander #{Expander.version}"
+            exit 0
+          end
+
+        end
+
+        def self.parse_options(argv)
+          @option_parser.parse!(argv.dup)
+        end
+
+        def self.config
+          @config
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/chef/expander/control.rb

@@ -0,0 +1,206 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'bunny'
+require 'yajl'
+require 'eventmachine'
+require 'amqp'
+require 'mq'
+require 'highline'
+
+require 'chef/expander/node'
+require 'chef/expander/configuration'
+
+require 'pp'
+
+module Chef
+  module Expander
+    class Control
+
+      def self.run(argv)
+        remaining_args_after_opts = Expander.init_config(ARGV)
+        new(remaining_args_after_opts).run
+      end
+
+      def self.desc(description)
+        @desc = description
+      end
+
+      def self.option(*args)
+        #TODO
+      end
+
+      def self.arg(*args)
+        #TODO
+      end
+
+      def self.descriptions
+        @descriptions ||= []
+      end
+
+      def self.method_added(method_name)
+        if @desc
+          descriptions << [method_name, method_name.to_s.gsub('_', '-'), @desc]
+          @desc = nil
+        end
+      end
+
+      #--
+      # TODO: this is confusing and unneeded. Just whitelist the methods
+      # that map to commands and use +send+
+      def self.compile
+        run_method = "def run; case @argv.first;"
+        descriptions.each do |method_name, command_name, desc|
+          run_method << "when '#{command_name}';#{method_name};"
+        end
+        run_method << "else; help; end; end;"
+        class_eval(run_method, __FILE__, __LINE__)
+      end
+
+      def initialize(argv)
+        @argv = argv.dup
+      end
+
+      desc "Show this message"
+      def help
+        puts "Chef Expander #{Expander.version}"
+        puts "Usage: chef-expanderctl COMMAND"
+        puts
+        puts "Commands:"
+        self.class.descriptions.each do |method_name, command_name, desc|
+          puts "  #{command_name}".ljust(15) + desc
+        end
+      end
+
+      desc "display the aggregate queue backlog"
+      def queue_depth
+        h = HighLine.new
+        message_counts = []
+
+        amqp_client = Bunny.new(Expander.config.amqp_config)
+        amqp_client.start
+
+        0.upto(VNODES - 1) do |vnode|
+          q = amqp_client.queue("vnode-#{vnode}", :durable => true)
+          message_counts << q.status[:message_count]
+        end
+        total_messages = message_counts.inject(0) { |sum, count| sum + count }
+        max = message_counts.max
+        min = message_counts.min
+
+        avg = total_messages.to_f / message_counts.size.to_f
+
+        puts "  total messages:       #{total_messages}"
+        puts "  average queue depth:  #{avg}"
+        puts "  max queue depth:      #{max}"
+        puts "  min queue depth:      #{min}" 
+      ensure
+        amqp_client.stop if defined?(amqp_client) && amqp_client
+      end
+
+      desc "show the backlog and consumer count for each vnode queue"
+      def queue_status
+        h = HighLine.new
+        queue_status = [h.color("VNode", :bold), h.color("Messages", :bold), h.color("Consumers", :bold)]
+
+        total_messages = 0
+
+        amqp_client = Bunny.new(Expander.config.amqp_config)
+        amqp_client.start
+
+        0.upto(VNODES - 1) do |vnode|
+          q = amqp_client.queue("vnode-#{vnode}", :durable => true)
+          status = q.status
+          # returns {:message_count => method.message_count, :consumer_count => method.consumer_count}
+          queue_status << vnode.to_s << status[:message_count].to_s << status[:consumer_count].to_s
+          total_messages += status[:message_count]
+        end
+        puts "  total messages: #{total_messages}"
+        puts
+        puts h.list(queue_status, :columns_across, 3)
+      ensure
+        amqp_client.stop if defined?(amqp_client) && amqp_client
+      end
+
+      desc "show the status of the nodes in the cluster"
+      def node_status
+        status_mutex = Mutex.new
+        h = ::HighLine.new
+        node_status = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Vnodes", :bold)]
+
+        print("Collecting status info from the cluster...")
+
+        AMQP.start(Expander.config.amqp_config) do
+          node = Expander::Node.local_node
+          node.exclusive_control_queue.subscribe do |header, message|
+            status = Yajl::Parser.parse(message)
+            status_mutex.synchronize do
+              node_status << status["hostname_f"]
+              node_status << status["pid"].to_s
+              node_status << status["guid"]
+              # BIG ASSUMPTION HERE that nodes only have contiguous vnode ranges
+              # will not be true once vnode recovery is implemented
+              node_status << "#{status["vnodes"].min}-#{status["vnodes"].max}"
+            end
+          end
+          node.broadcast_message(Yajl::Encoder.encode(:action => :status, :rsvp => node.exclusive_control_queue_name))
+          EM.add_timer(2) { AMQP.stop;EM.stop }
+        end
+
+        puts "done"
+        puts
+        puts h.list(node_status, :columns_across, 4)
+        puts
+      end
+
+      desc "sets the log level of all nodes in the cluster"
+      def log_level
+        @argv.shift
+        level = @argv.first
+        acceptable_levels = %w{debug info warn error fatal}
+        unless acceptable_levels.include?(level)
+          puts "Log level must be one of #{acceptable_levels.join(', ')}"
+          exit 1
+        end
+
+        h = HighLine.new
+        response_mutex = Mutex.new
+        
+        responses = [h.color("Host", :bold), h.color("PID", :bold), h.color("GUID", :bold), h.color("Log Level", :bold)]
+        AMQP.start(Expander.config.amqp_config) do
+          node = Expander::Node.local_node
+          node.exclusive_control_queue.subscribe do |header, message|
+            reply = Yajl::Parser.parse(message)
+            n = reply['node']
+            response_mutex.synchronize do
+              responses << n["hostname_f"] << n["pid"].to_s << n["guid"] << reply["level"]
+            end
+          end
+          node.broadcast_message(Yajl::Encoder.encode({:action => :set_log_level, :level => level, :rsvp => node.exclusive_control_queue_name}))
+          EM.add_timer(2) { AMQP.stop; EM.stop }
+        end
+        puts h.list(responses, :columns_across, 4)
+      end
+
+
+      compile
+    end
+  end
+end

data/lib/chef/expander/daemonizable.rb

@@ -0,0 +1,150 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'etc'
+require 'chef/expander/loggable'
+
+module Chef
+  module Expander
+
+    class AlreadyRunning < RuntimeError
+    end
+
+    class NoSuchUser < ArgumentError
+    end
+
+    class NoSuchGroup < ArgumentError
+    end
+
+    module Daemonizable
+      include Loggable
+
+      # Daemonizes the process if configured to do so, and ensures that only one
+      # copy of the process is running with a given config by obtaining an
+      # exclusive lock on the pidfile. Also sets process user and group if so
+      # configured.
+      # ===Raises
+      # * AlreadyRunning::: when another process has the exclusive lock on the pidfile
+      # * NoSuchUser::: when a user is configured that doesn't exist
+      # * NoSuchGroup::: when a group is configured that doesn't exist
+      # * SystemCallError::: if there is an error creating the pidfile
+      def configure_process
+        Expander.config.daemonize? ? daemonize : ensure_exclusive
+        set_user_and_group
+      end
+
+      def daemonize
+        acquire_locks
+        exit if fork
+        Process.setsid
+        exit if fork
+        write_pid
+        Dir.chdir('/')
+        STDIN.reopen("/dev/null")
+        STDOUT.reopen("/dev/null", "a")
+        STDERR.reopen("/dev/null", "a")
+      end
+
+      # When not forking into the background, this ensures only one chef-expander
+      # is running with a given config and writes the process id to the pidfile.
+      def ensure_exclusive
+        acquire_locks
+        write_pid
+      end
+
+      def set_user_and_group
+        return nil if Expander.config.user.nil?
+
+        if Expander.config.group.nil?
+          log.info {"Changing user to #{Expander.config.user}"}
+        else
+          log.info {"Changing user to #{Expander.config.user} and group to #{Expander.config.group}"}
+        end
+
+        unless (set_group && set_user)
+          log.error {"Unable to change user to #{Expander.config.user} - Are you root?"}
+        end
+      end
+
+      # Deletes the pidfile, releasing the exclusive lock on it in the process.
+      def release_locks
+        File.unlink(@pidfile.path) if File.exist?(@pidfile.path)
+        @pidfile.close unless @pidfile.closed?
+      end
+
+      private
+
+      def set_user
+        Process::Sys.setuid(target_uid)
+        true
+      rescue Errno::EPERM => e
+        log.debug {e}
+        false
+      end
+
+      def set_group
+        if gid = target_uid
+          Process::Sys.setgid(gid)
+        end
+        true
+      rescue Errno::EPERM
+        log.debug {e}
+        false
+      end
+
+      def target_uid
+        user = Expander.config.user
+        user.kind_of?(Fixnum) ? user : Etc.getpwnam(user).uid
+      rescue ArgumentError => e
+        log.debug {e}
+        raise NoSuchUser, "Cannot change user to #{user} - failed to find the uid"
+      end
+
+      def target_gid
+        if group = Expander.config.group
+          group.kind_of?(Fixnum) ? group : Etc.getgrnam(group).gid
+        else
+          nil
+        end
+      rescue ArgumentError => e
+        log.debug {e}
+        raise NoSuchGroup, "Cannot change group to #{group} - failed to find the gid"
+      end
+
+      def acquire_locks
+        @pidfile = File.open(Expander.config.pidfile, File::RDWR|File::CREAT, 0644)
+        unless @pidfile.flock(File::LOCK_EX | File::LOCK_NB)
+          pid = @pidfile.read.strip
+          msg = "Another instance of chef-expander (pid: #{pid}) has a lock on the pidfile (#{Expander.config.pidfile}). \n"\
+                "Configure a different pidfile to run multiple instances of chef-expander at once."
+          raise AlreadyRunning, msg
+        end
+      rescue Exception
+        @pidfile.close if @pidfile && !@pidfile.closed?
+        raise
+      end
+
+      def write_pid
+        @pidfile.truncate(0)
+        @pidfile.print("#{Process.pid}\n")
+        @pidfile.flush
+      end
+
+    end
+  end
+end