chef-expander 0.10.0.beta.0

data/lib/chef/expander/version.rb

@@ -0,0 +1,37 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'open3'
+
+module Chef
+  module Expander
+
+    VERSION = "0.10.0.beta.0"
+
+    def self.version
+      @rev ||= begin
+        rev = Open3.popen3("git rev-parse HEAD") {|stdin, stdout, stderr| stdout.read }.strip
+        rev.empty? ? nil : " (#{rev})"
+      end
+      "#{VERSION}#@rev"
+    end
+
+  end
+end

data/lib/chef/expander/vnode.rb

@@ -0,0 +1,106 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'eventmachine'
+require 'amqp'
+require 'mq'
+
+require 'chef/expander/loggable'
+require 'chef/expander/solrizer'
+
+module Chef
+  module Expander
+    class VNode
+      include Loggable
+
+      attr_reader :vnode_number
+
+      attr_reader :supervise_interval
+
+      def initialize(vnode_number, supervisor, opts={})
+        @vnode_number = vnode_number.to_i
+        @supervisor   = supervisor
+        @queue    = nil
+        @stopped  = false
+        @supervise_interval = opts[:supervise_interval] || 30
+      end
+
+      def start
+        @supervisor.vnode_added(self)
+
+        subscription_confirmed = Proc.new do
+          abort_on_multiple_subscribe
+          supervise_consumer_count
+        end
+
+        queue.subscribe(:ack => true, :confirm => subscription_confirmed) do |headers, payload|
+          log.debug {"got #{payload} size(#{payload.size} bytes) on queue #{queue_name}"}
+          solrizer = Solrizer.new(payload) { headers.ack }
+          solrizer.run
+        end
+
+      rescue MQ::Error => e
+        log.error {"Failed to start subscriber on #{queue_name} #{e.class.name}: #{e.message}"}
+      end
+
+      def supervise_consumer_count
+        EM.add_periodic_timer(supervise_interval) do
+          abort_on_multiple_subscribe
+        end
+      end
+
+      def abort_on_multiple_subscribe
+        queue.status do |message_count, subscriber_count|
+          if subscriber_count.to_i > 1
+            log.error { "Detected extra consumers (#{subscriber_count} total) on queue #{queue_name}, cancelling subscription" }
+            stop
+          end
+        end
+      end
+
+      def stop
+        log.debug {"Cancelling subscription on queue #{queue_name.inspect}"}
+        queue.unsubscribe if queue.subscribed?
+        @supervisor.vnode_removed(self)
+        @stopped = true
+      end
+
+      def stopped?
+        @stopped
+      end
+
+      def queue
+        @queue ||= begin
+          log.debug { "declaring queue #{queue_name}" }
+          MQ.queue(queue_name, :passive => false, :durable => true)
+        end
+      end
+
+      def queue_name
+        "vnode-#{@vnode_number}"
+      end
+
+      def control_queue_name
+        "#{queue_name}-control"
+      end
+
+    end
+  end
+end

data/lib/chef/expander/vnode_supervisor.rb

@@ -0,0 +1,265 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'yajl'
+require 'eventmachine'
+require 'amqp'
+require 'mq'
+require 'chef/expander/version'
+require 'chef/expander/loggable'
+require 'chef/expander/node'
+require 'chef/expander/vnode'
+require 'chef/expander/vnode_table'
+require 'chef/expander/configuration'
+
+module ::AMQP
+  def self.hard_reset!
+    MQ.reset rescue nil
+    stop
+    EM.stop rescue nil
+    Thread.current[:mq], @conn = nil, nil
+  end
+end
+
+module Chef
+  module Expander
+    class VNodeSupervisor
+      include Loggable
+      extend  Loggable
+
+      COULD_NOT_CONNECT = /Could not connect to server/.freeze
+
+      def self.start_cluster_worker
+        @vnode_supervisor = new
+        @original_ppid = Process.ppid
+        trap_signals
+
+        vnodes = Expander.config.vnode_numbers
+
+        $0 = "chef-expander#{Expander.config.ps_tag} worker ##{Expander.config.index} (vnodes #{vnodes.min}-#{vnodes.max})"
+        
+        AMQP.start(Expander.config.amqp_config) do
+          start_consumers
+          await_parent_death
+        end
+      end
+
+      def self.await_parent_death
+        @awaiting_parent_death = EM.add_periodic_timer(1) do
+          unless Process.ppid == @original_ppid
+            @awaiting_parent_death.cancel
+            stop("master process death")
+          end
+        end
+      end
+
+      def self.start
+        @vnode_supervisor = new
+        trap_signals
+
+        Expander.init_config(ARGV)
+
+        log.info("Chef Search Expander #{Expander.version} starting up.")
+
+        begin
+          AMQP.start(Expander.config.amqp_config) do
+            start_consumers
+          end
+        rescue AMQP::Error => e
+          if e.message =~ COULD_NOT_CONNECT
+            log.error { "Could not connect to rabbitmq. Make sure it is running and correctly configured." }
+            log.error { e.message }
+
+            AMQP.hard_reset!
+
+            sleep 5
+            retry
+          else
+            raise
+          end
+        end
+      end
+
+      def self.start_consumers
+        log.debug { "Setting prefetch count to 1"}
+        MQ.prefetch(1)
+
+        vnodes = Expander.config.vnode_numbers
+        log.info("Starting Consumers for vnodes #{vnodes.min}-#{vnodes.max}")
+        @vnode_supervisor.start(vnodes)
+      end
+
+      def self.trap_signals
+        Kernel.trap(:INT)  { stop_immediately(:INT) }
+        Kernel.trap(:TERM) { stop_gracefully(:TERM) }
+      end
+
+      def self.stop_immediately(signal)
+        log.info { "Initiating immediate shutdown on signal (#{signal})" }
+        @vnode_supervisor.stop
+        EM.add_timer(1) do
+          AMQP.stop
+          EM.stop
+        end
+      end
+
+      def self.stop_gracefully(signal)
+        log.info { "Initiating graceful shutdown on signal (#{signal})" }
+        @vnode_supervisor.stop
+        wait_for_http_requests_to_complete
+      end
+
+      def self.wait_for_http_requests_to_complete
+        if Expander::Solrizer.http_requests_active?
+          log.info { "waiting for in progress HTTP Requests to complete"}
+          EM.add_timer(1) do
+            wait_for_http_requests_to_complete
+          end
+        else
+          log.info { "HTTP requests completed, shutting down"}
+          AMQP.stop
+          EM.stop
+        end
+      end
+
+      attr_reader :vnode_table
+
+      attr_reader :local_node
+
+      def initialize
+        @vnodes = {}
+        @vnode_table = VNodeTable.new(self)
+        @local_node  = Node.local_node
+        @queue_name, @guid = nil, nil
+      end
+
+      def start(vnode_ids)
+        @local_node.start do |message|
+          process_control_message(message)
+        end
+
+        #start_vnode_table_publisher
+
+        Array(vnode_ids).each { |vnode_id| spawn_vnode(vnode_id) }
+      end
+
+      def stop
+        @local_node.stop
+
+        #log.debug { "stopping vnode table updater" }
+        #@vnode_table_publisher.cancel
+
+        log.info { "Stopping VNode queue subscribers"}
+        @vnodes.each do |vnode_number, vnode|
+          log.debug { "Stopping consumer on VNode #{vnode_number}"}
+          vnode.stop
+        end
+        
+      end
+
+      def vnode_added(vnode)
+        log.debug { "vnode #{vnode.vnode_number} registered with supervisor" }
+        @vnodes[vnode.vnode_number.to_i] = vnode
+      end
+
+      def vnode_removed(vnode)
+        log.debug { "vnode #{vnode.vnode_number} unregistered from supervisor" }
+        @vnodes.delete(vnode.vnode_number.to_i)
+      end
+
+      def vnodes
+        @vnodes.keys.sort
+      end
+
+      def spawn_vnode(vnode_number)
+        VNode.new(vnode_number, self).start
+      end
+
+      def release_vnode
+        # TODO
+      end
+
+      def process_control_message(message)
+        control_message = parse_symbolic(message)
+        case control_message[:action]
+        when "claim_vnode"
+          spawn_vnode(control_message[:vnode_id])
+        when "recover_vnode"
+          recover_vnode(control_message[:vnode_id])
+        when "release_vnodes"
+          raise "todo"
+          release_vnode()
+        when "update_vnode_table"
+          @vnode_table.update_table(control_message[:data])
+        when "vnode_table_publish"
+          publish_vnode_table
+        when "status"
+          publish_status_to(control_message[:rsvp])
+        when "set_log_level"
+          set_log_level(control_message[:level], control_message[:rsvp])
+        else
+          log.error { "invalid control message #{control_message.inspect}" }
+        end
+      rescue Exception => e
+        log.error { "Error processing a control message."}
+        log.error { "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}" }
+      end
+
+
+      def start_vnode_table_publisher
+        @vnode_table_publisher = EM.add_periodic_timer(10) { publish_vnode_table }
+      end
+
+      def publish_vnode_table
+        status_update = @local_node.to_hash
+        status_update[:vnodes] = vnodes
+        status_update[:update] = :add
+        @local_node.broadcast_message(Yajl::Encoder.encode({:action => :update_vnode_table, :data => status_update}))
+      end
+
+      def publish_status_to(return_queue)
+        status_update = @local_node.to_hash
+        status_update[:vnodes] = vnodes
+        MQ.queue(return_queue).publish(Yajl::Encoder.encode(status_update))
+      end
+
+      def set_log_level(level, rsvp_to)
+        log.info { "setting log level to #{level} due to command from #{rsvp_to}" }
+        new_log_level = (Expander.config.log_level = level.to_sym)
+        reply = {:level => new_log_level, :node => @local_node.to_hash}
+        MQ.queue(rsvp_to).publish(Yajl::Encoder.encode(reply))
+      end
+
+      def recover_vnode(vnode_id)
+        if @vnode_table.local_node_is_leader?
+          log.debug { "Recovering vnode: #{vnode_id}" }
+          @local_node.shared_message(Yajl::Encoder.encode({:action => :claim_vnode, :vnode_id => vnode_id}))
+        else
+          log.debug { "Ignoring :recover_vnode message because this node is not the leader" }
+        end
+      end
+
+      def parse_symbolic(message)
+        Yajl::Parser.new(:symbolize_keys => true).parse(message)
+      end
+
+    end
+  end
+end

data/lib/chef/expander/vnode_table.rb

@@ -0,0 +1,83 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'yajl'
+require 'chef/expander/node'
+require 'chef/expander/loggable'
+
+module Chef
+  module Expander
+    class VNodeTable
+
+      include Loggable
+
+      class InvalidVNodeTableUpdate < ArgumentError; end
+
+      attr_reader :vnodes_by_node
+
+      def initialize(vnode_supervisor)
+        @node_update_mutex = Mutex.new
+        @vnode_supervisor = vnode_supervisor
+        @vnodes_by_node = {}
+      end
+
+      def nodes
+        @vnodes_by_node.keys
+      end
+
+      def update_table(table_update)
+        case table_update[:update]
+        when "add", "update"
+          update_node(table_update)
+        when "remove"
+          remove_node(table_update)
+        else
+          raise InvalidVNodeTableUpdate, "no action or action not acceptable: #{table_update.inspect}"
+        end
+        log.debug { "current vnode table: #{@vnodes_by_node.inspect}" }
+      end
+
+      def update_node(node_info)
+       @node_update_mutex.synchronize do
+          @vnodes_by_node[Node.from_hash(node_info)] = node_info[:vnodes]
+        end
+      end
+
+      def remove_node(node_info)
+        @node_update_mutex.synchronize do
+          @vnodes_by_node.delete(Node.from_hash(node_info))
+        end
+      end
+
+      def leader_node
+        if @vnodes_by_node.empty?
+          nil
+        else
+          Array(@vnodes_by_node).reject { |node| node[1].empty? }.sort { |a,b| a[1].min <=> b[1].min }.first[0]
+        end
+      end
+
+      def local_node_is_leader?
+        (Node.local_node == leader_node) || (@vnodes_by_node[Node.local_node].include?(0))
+      end
+
+    end
+  end
+end

data/lib/chef/expander.rb

@@ -0,0 +1,36 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Chef
+  module Expander
+
+    # VNODES is the number of queues in rabbit that are available for subscribing.
+    # The name comes from riak, where the data ring (160bits) is chunked into
+    # many vnodes; vnodes outnumber physical nodes, so one node hosts several
+    # vnodes. That is the same design we use here.
+    #
+    # See the notes on topic queue benchmarking before adjusting this value.
+    VNODES = 1024
+
+    SHARED_CONTROL_QUEUE_NAME = "chef-search-control--shared"
+    BROADCAST_CONTROL_EXCHANGE_NAME = 'chef-search-control--broadcast'
+
+  end
+end

data/scripts/check_queue_size

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+#--
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###############################################################################
+# check_queue_size
+# A Nagios Check for Chef Server Queue Backlogs
+###############################################################################
+
+require 'rubygems'
+
+Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do
+
+  require 'bunny'
+
+  $:.unshift(File.expand_path('./lib'))
+  require 'chef/expander'
+  require 'chef/expander/version'
+  require 'chef/expander/configuration'
+
+  include Chef
+
+  Expander.init_config([])
+
+  config = {:warn => 100, :crit => 200}
+
+  option_parser = OptionParser.new do |o|
+    o.banner = "Usage: check_queue_size [options]"
+
+    o.on('-w', '--warn WARN_THRESHOLD', 'number of messages to trigger a warning') do |i|
+      config[:warn] = i.to_i
+    end
+
+    o.on('-c', '--critical CRITICAL_THRESHOLD', 'the number of messages to trigger a critical') do |n|
+      config[:crit] = n.to_i
+    end
+
+    o.on_tail('-h', '--help', 'show this message') do
+      puts "chef-expander #{Expander.version}"
+      puts "queue size monitor"
+      puts ''
+      puts o
+      exit 127
+    end
+  end
+
+  option_parser.parse!(ARGV.dup)
+
+  message_counts = []
+
+  begin
+    amqp_client = Bunny.new(Expander.config.amqp_config)
+    amqp_client.start
+
+    0.upto(Expander::VNODES - 1) do |vnode|
+      q = amqp_client.queue("vnode-#{vnode}", :durable => true)
+      message_counts << q.status[:message_count]
+    end
+    total_messages = message_counts.inject(:+)
+
+    if total_messages >= config[:crit]
+      puts "Chef Expander Queue Size CRITICAL - messages: #{total_messages}"
+      exit(2)
+    elsif total_messages >= config[:warn]
+      puts "Chef Expander Queue Size WARNING - messages: #{total_messages}"
+      exit(1)
+    else
+      puts "Chef Expander Queue Size OK - messages: #{total_messages}"
+      exit(0)
+    end
+
+  ensure
+    amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected?
+  end
+  
+end

data/scripts/check_queue_size_munin

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'munin_plugin'
+Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do
+  require 'bunny'
+
+  $:.unshift(File.expand_path('../../lib', __FILE__))
+  require 'chef/expander'
+  require 'chef/expander/version'
+  require 'chef/expander/configuration'
+
+  include Chef
+
+munin_plugin do
+  graph_title "Expander Queue Size"
+  graph_vlabel "Events"
+  graph_category "Opscode"
+  graph_info "Events in the Expander Queue waiting to be consumed by Solr."
+  queuesize.label "events"
+  queuesize.draw "LINE"
+  queuesize.warning "100"
+  queuesize.critical "200"
+
+  collect do
+
+      Expander.init_config([])
+
+      message_counts = []
+
+      begin
+        amqp_client = Bunny.new(Expander.config.amqp_config)
+        amqp_client.start
+
+        0.upto(Expander::VNODES - 1) do |vnode|
+          q = amqp_client.queue("vnode-#{vnode}", :durable => true)
+          message_counts << q.status[:message_count]
+        end
+        total_messages = message_counts.inject(:+)
+
+      ensure
+        amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected?
+      end
+
+      queuesize.value total_messages
+  
+    end
+
+
+  end
+end