chef-expander 0.10.0.beta.0

data/scripts/make_solr_xml

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "rubygems"
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'yajl'
+require 'chef/expander/solrizer'
+
+USAGE = <<-EOH
+#{$0} [file ...]
+
+Convert Chef object JSON files into XML documents of the same name but
+with a ".xml" extension containing the XML used for Solr indexing.
+EOH
+
+if ARGV.size == 0
+  abort USAGE
+end
+
+ARGV.each do |obj_file|
+  raw_json = open(obj_file, "r").read
+  item_json = Yajl::Parser.parse(raw_json)
+  payload = {
+    :item => item_json,
+    :type => item_json["chef_type"].to_s,
+    :database => "riak_search_test",
+    :id => item_json["name"],
+    :enqueued_at => Time.now.to_i
+  }
+  update_obj = {:action => "add", :payload => payload}
+  update_json = Yajl::Encoder.encode(update_obj)
+  solrizer = Chef::Expander::Solrizer.new(update_json) { :no_op }
+  solrizer.log.init(StringIO.new)
+
+  out = File.basename(obj_file).sub(/\.json$/, "") + ".xml"
+  open(out, "w") do |f|
+    f.write(solrizer.pointyize_add)
+  end
+end

data/scripts/traffic-creator

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Author:: Chris Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2010-2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "rubygems"
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'pp'
+require 'bunny'
+require 'yajl'
+require 'uuidtools'
+require 'word_salad'
+
+require 'chef_search/expander/configuration'
+
+Chef::Expander.init_config(ARGV)
+
+MESSAGES_TO_SEND = 10_000
+
+NUM_RAND_KEY_PAIRS = 50
+NUM_RAND_VALUE_PAIRS = 50
+
+PERSISTENT_MESSAGES = true
+
+KEYS = NUM_RAND_VALUE_PAIRS.words
+
+SAMPLE_NODES = []
+Dir.glob(File.expand_path(File.dirname(__FILE__)) + '/../data/*_node.json') do |node_file|
+  SAMPLE_NODES << Yajl::Parser.parse(IO.read(node_file))
+end
+
+NUM_NODES = SAMPLE_NODES.size
+puts "Read #{NUM_NODES} sample nodes"
+
+puts "Using rabbitmq config #{Chef::Expander.config.amqp_config.inspect}"
+
+puts "connecting to rabbitmq"
+amqp_client = Bunny.new(Chef::Expander.config.amqp_config)
+amqp_client.start
+
+puts 'declaring queues'
+queues = {}
+0.upto(1023) do |vnode|
+  queues[vnode] = amqp_client.queue("vnode-#{vnode}", :durable => true)
+end
+  
+def add_rand_keys(node)
+  rand_key_vals = Hash[*((2 * NUM_RAND_KEY_PAIRS).words)]
+  rand_vals = Hash[*(KEYS.zip(NUM_RAND_VALUE_PAIRS.words)).flatten]
+  node.merge(rand_key_vals.merge(rand_vals))
+end
+
+puts "sending #{MESSAGES_TO_SEND} messages"
+start_time = Time.now
+sent_messages = 0
+1.upto(MESSAGES_TO_SEND) do
+  node = SAMPLE_NODES[rand(NUM_NODES)]
+  node = add_rand_keys(node)
+  index_data = {:action => :add}
+  index_data[:payload] = {:item => node}
+  index_data[:payload][:type] = :node
+  index_data[:payload][:database] = :testdb
+  index_data[:payload][:enqueued_at] = Time.now.utc.to_i
+
+  id = node["name"]
+  vnode = rand(1024)
+  index_data[:payload][:id] = id
+
+  puts "queue: vnode-#{vnode} (#{sent_messages} / #{MESSAGES_TO_SEND})"
+  amqp_client.tx_select if PERSISTENT_MESSAGES
+  queues[vnode].publish(Yajl::Encoder.encode(index_data), :persistent => PERSISTENT_MESSAGES)
+  amqp_client.tx_commit if PERSISTENT_MESSAGES
+  sent_messages += 1
+end
+end_time = Time.now
+
+total_time = end_time - start_time
+rate = MESSAGES_TO_SEND.to_f / total_time
+puts "done (#{total_time}s, #{rate} msg/s)"

metadata

@@ -0,0 +1,202 @@
+--- !ruby/object:Gem::Specification 
+name: chef-expander
+version: !ruby/object:Gem::Version 
+  prerelease: 7
+  version: 0.10.0.beta.0
+platform: ruby
+authors: 
+- Adam Jacob
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-28 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mixlib-log
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.6.7
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.12.10
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.2.11
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: yajl-ruby
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.7.7
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: uuidtools
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 2.1.1
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: fast_xs
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.7.3
+  type: :runtime
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: highline
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.6.1
+  type: :runtime
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.8.7
+  type: :runtime
+  version_requirements: *id010
+- !ruby/object:Gem::Dependency 
+  name: bunny
+  prerelease: false
+  requirement: &id011 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.0
+  type: :runtime
+  version_requirements: *id011
+description: A systems integration framework, built to bring the benefits of configuration management to your entire infrastructure.
+email: adam@opscode.com
+executables: 
+- chef-expander
+- chef-expander-cluster
+- chef-expanderctl
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- LICENSE
+- README.rdoc
+- scripts/check_queue_size
+- scripts/check_queue_size_munin
+- scripts/make_solr_xml
+- scripts/traffic-creator
+- conf/chef-expander.rb.example
+- lib/chef/expander/cluster_supervisor.rb
+- lib/chef/expander/configuration.rb
+- lib/chef/expander/control.rb
+- lib/chef/expander/daemonizable.rb
+- lib/chef/expander/flattener.rb
+- lib/chef/expander/loggable.rb
+- lib/chef/expander/logger.rb
+- lib/chef/expander/node.rb
+- lib/chef/expander/solrizer.rb
+- lib/chef/expander/version.rb
+- lib/chef/expander/vnode.rb
+- lib/chef/expander/vnode_supervisor.rb
+- lib/chef/expander/vnode_table.rb
+- lib/chef/expander.rb
+- bin/chef-expander
+- bin/chef-expander-cluster
+- bin/chef-expanderctl
+has_rdoc: true
+homepage: http://wiki.opscode.com/display/chef
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 1.3.1
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: A systems integration framework, built to bring the benefits of configuration management to your entire infrastructure.
+test_files: []
+