erchef-expander 11.4.0

data/lib/chef/expander/vnode_table.rb

@@ -0,0 +1,83 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'yajl'
+require 'chef/expander/node'
+require 'chef/expander/loggable'
+
+module Chef
+  module Expander
+    class VNodeTable
+
+      include Loggable
+
+      class InvalidVNodeTableUpdate < ArgumentError; end
+
+      attr_reader :vnodes_by_node
+
+      def initialize(vnode_supervisor)
+        @node_update_mutex = Mutex.new
+        @vnode_supervisor = vnode_supervisor
+        @vnodes_by_node = {}
+      end
+
+      def nodes
+        @vnodes_by_node.keys
+      end
+
+      def update_table(table_update)
+        case table_update[:update]
+        when "add", "update"
+          update_node(table_update)
+        when "remove"
+          remove_node(table_update)
+        else
+          raise InvalidVNodeTableUpdate, "no action or action not acceptable: #{table_update.inspect}"
+        end
+        log.debug { "current vnode table: #{@vnodes_by_node.inspect}" }
+      end
+
+      def update_node(node_info)
+       @node_update_mutex.synchronize do
+          @vnodes_by_node[Node.from_hash(node_info)] = node_info[:vnodes]
+        end
+      end
+
+      def remove_node(node_info)
+        @node_update_mutex.synchronize do
+          @vnodes_by_node.delete(Node.from_hash(node_info))
+        end
+      end
+
+      def leader_node
+        if @vnodes_by_node.empty?
+          nil
+        else
+          Array(@vnodes_by_node).reject { |node| node[1].empty? }.sort { |a,b| a[1].min <=> b[1].min }.first[0]
+        end
+      end
+
+      def local_node_is_leader?
+        (Node.local_node == leader_node) || (@vnodes_by_node[Node.local_node].include?(0))
+      end
+
+    end
+  end
+end

data/scripts/check_queue_size

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+#--
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###############################################################################
+# check_queue_size
+# A Nagios Check for Chef Server Queue Backlogs
+###############################################################################
+
+require 'rubygems'
+
+Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do
+
+  require 'bunny'
+
+  $:.unshift(File.expand_path('./lib'))
+  require 'chef/expander'
+  require 'chef/expander/version'
+  require 'chef/expander/configuration'
+
+  include Chef
+
+  Expander.init_config([])
+
+  config = {:warn => 100, :crit => 200}
+
+  option_parser = OptionParser.new do |o|
+    o.banner = "Usage: check_queue_size [options]"
+
+    o.on('-w', '--warn WARN_THRESHOLD', 'number of messages to trigger a warning') do |i|
+      config[:warn] = i.to_i
+    end
+
+    o.on('-c', '--critical CRITICAL_THRESHOLD', 'the number of messages to trigger a critical') do |n|
+      config[:crit] = n.to_i
+    end
+
+    o.on_tail('-h', '--help', 'show this message') do
+      puts "chef-expander #{Expander.version}"
+      puts "queue size monitor"
+      puts ''
+      puts o
+      exit 127
+    end
+  end
+
+  option_parser.parse!(ARGV.dup)
+
+  message_counts = []
+
+  begin
+    amqp_client = Bunny.new(Expander.config.amqp_config)
+    amqp_client.start
+
+    0.upto(Expander::VNODES - 1) do |vnode|
+      q = amqp_client.queue("vnode-#{vnode}", :durable => true)
+      message_counts << q.status[:message_count]
+    end
+    total_messages = message_counts.inject(:+)
+
+    if total_messages >= config[:crit]
+      puts "Chef Expander Queue Size CRITICAL - messages: #{total_messages}"
+      exit(2)
+    elsif total_messages >= config[:warn]
+      puts "Chef Expander Queue Size WARNING - messages: #{total_messages}"
+      exit(1)
+    else
+      puts "Chef Expander Queue Size OK - messages: #{total_messages}"
+      exit(0)
+    end
+
+  ensure
+    amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected?
+  end
+  
+end

data/scripts/check_queue_size_munin

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'munin_plugin'
+Dir.chdir(File.join(File.expand_path(File.dirname(__FILE__)), "..")) do
+  require 'bunny'
+
+  $:.unshift(File.expand_path('../../lib', __FILE__))
+  require 'chef/expander'
+  require 'chef/expander/version'
+  require 'chef/expander/configuration'
+
+  include Chef
+
+munin_plugin do
+  graph_title "Expander Queue Size"
+  graph_vlabel "Events"
+  graph_category "Opscode"
+  graph_info "Events in the Expander Queue waiting to be consumed by Solr."
+  queuesize.label "events"
+  queuesize.draw "LINE"
+  queuesize.warning "100"
+  queuesize.critical "200"
+
+  collect do
+
+      Expander.init_config([])
+
+      message_counts = []
+
+      begin
+        amqp_client = Bunny.new(Expander.config.amqp_config)
+        amqp_client.start
+
+        0.upto(Expander::VNODES - 1) do |vnode|
+          q = amqp_client.queue("vnode-#{vnode}", :durable => true)
+          message_counts << q.status[:message_count]
+        end
+        total_messages = message_counts.inject(:+)
+
+      ensure
+        amqp_client.stop if defined?(amqp_client) && amqp_client && amqp_client.connected?
+      end
+
+      queuesize.value total_messages
+  
+    end
+
+
+  end
+end

data/scripts/make_solr_xml

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "rubygems"
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'yajl'
+require 'chef/expander/solrizer'
+
+USAGE = <<-EOH
+#{$0} [file ...]
+
+Convert Chef object JSON files into XML documents of the same name but
+with a ".xml" extension containing the XML used for Solr indexing.
+EOH
+
+if ARGV.size == 0
+  abort USAGE
+end
+
+ARGV.each do |obj_file|
+  raw_json = open(obj_file, "r").read
+  item_json = Yajl::Parser.parse(raw_json)
+  payload = {
+    :item => item_json,
+    :type => item_json["chef_type"].to_s,
+    :database => "riak_search_test",
+    :id => item_json["name"],
+    :enqueued_at => Time.now.to_i
+  }
+  update_obj = {:action => "add", :payload => payload}
+  update_json = Yajl::Encoder.encode(update_obj)
+  solrizer = Chef::Expander::Solrizer.new(update_json) { :no_op }
+  solrizer.log.init(StringIO.new)
+
+  out = File.basename(obj_file).sub(/\.json$/, "") + ".xml"
+  open(out, "w") do |f|
+    f.write(solrizer.pointyize_add)
+  end
+end

data/scripts/traffic-creator

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "rubygems"
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'pp'
+require 'bunny'
+require 'yajl'
+require 'uuidtools'
+require 'word_salad'
+
+require 'chef_search/expander/configuration'
+
+Chef::Expander.init_config(ARGV)
+
+MESSAGES_TO_SEND = 10_000
+
+NUM_RAND_KEY_PAIRS = 50
+NUM_RAND_VALUE_PAIRS = 50
+
+PERSISTENT_MESSAGES = true
+
+KEYS = NUM_RAND_VALUE_PAIRS.words
+
+SAMPLE_NODES = []
+Dir.glob(File.expand_path(File.dirname(__FILE__)) + '/../data/*_node.json') do |node_file|
+  SAMPLE_NODES << Yajl::Parser.parse(IO.read(node_file))
+end
+
+NUM_NODES = SAMPLE_NODES.size
+puts "Read #{NUM_NODES} sample nodes"
+
+puts "Using rabbitmq config #{Chef::Expander.config.amqp_config.inspect}"
+
+puts "connecting to rabbitmq"
+amqp_client = Bunny.new(Chef::Expander.config.amqp_config)
+amqp_client.start
+
+puts 'declaring queues'
+queues = {}
+0.upto(1023) do |vnode|
+  queues[vnode] = amqp_client.queue("vnode-#{vnode}", :durable => true)
+end
+  
+def add_rand_keys(node)
+  rand_key_vals = Hash[*((2 * NUM_RAND_KEY_PAIRS).words)]
+  rand_vals = Hash[*(KEYS.zip(NUM_RAND_VALUE_PAIRS.words)).flatten]
+  node.merge(rand_key_vals.merge(rand_vals))
+end
+
+puts "sending #{MESSAGES_TO_SEND} messages"
+start_time = Time.now
+sent_messages = 0
+1.upto(MESSAGES_TO_SEND) do
+  node = SAMPLE_NODES[rand(NUM_NODES)]
+  node = add_rand_keys(node)
+  index_data = {:action => :add}
+  index_data[:payload] = {:item => node}
+  index_data[:payload][:type] = :node
+  index_data[:payload][:database] = :testdb
+  index_data[:payload][:enqueued_at] = Time.now.utc.to_i
+
+  id = node["name"]
+  vnode = rand(1024)
+  index_data[:payload][:id] = id
+
+  puts "queue: vnode-#{vnode} (#{sent_messages} / #{MESSAGES_TO_SEND})"
+  amqp_client.tx_select if PERSISTENT_MESSAGES
+  queues[vnode].publish(Yajl::Encoder.encode(index_data), :persistent => PERSISTENT_MESSAGES)
+  amqp_client.tx_commit if PERSISTENT_MESSAGES
+  sent_messages += 1
+end
+end_time = Time.now
+
+total_time = end_time - start_time
+rate = MESSAGES_TO_SEND.to_f / total_time
+puts "done (#{total_time}s, #{rate} msg/s)"

metadata

@@ -0,0 +1,284 @@
+--- !ruby/object:Gem::Specification
+name: erchef-expander
+version: !ruby/object:Gem::Version
+  version: 11.4.0
+  prerelease: 
+platform: ruby
+authors:
+- Adam Jacob
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mixlib-log
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.10
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.11
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.1
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: fast_xs
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-expectations
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A systems integration framework, built to bring the benefits of configuration
+  management to your entire infrastructure.
+email: adam@opscode.com
+executables:
+- chef-expander
+- chef-expander-vnode
+- chef-expanderctl
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+- LICENSE
+files:
+- LICENSE
+- README.rdoc
+- scripts/check_queue_size
+- scripts/check_queue_size_munin
+- scripts/make_solr_xml
+- scripts/traffic-creator
+- conf/chef-expander.rb.example
+- lib/chef/expander/cluster_supervisor.rb
+- lib/chef/expander/configuration.rb
+- lib/chef/expander/control.rb
+- lib/chef/expander/daemonizable.rb
+- lib/chef/expander/flattener.rb
+- lib/chef/expander/loggable.rb
+- lib/chef/expander/logger.rb
+- lib/chef/expander/node.rb
+- lib/chef/expander/solrizer.rb
+- lib/chef/expander/version.rb
+- lib/chef/expander/vnode.rb
+- lib/chef/expander/vnode_supervisor.rb
+- lib/chef/expander/vnode_table.rb
+- lib/chef/expander.rb
+- bin/chef-expander
+- bin/chef-expander-vnode
+- bin/chef-expanderctl
+homepage: http://wiki.opscode.com/display/chef
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A systems integration framework, built to bring the benefits of configuration
+  management to your entire infrastructure.
+test_files: []
+has_rdoc: true