ciborg 0.0.1 → 3.0.0

data/chef/travis-cookbooks/ci_environment/riak/attributes/sysmon.rb

@@ -0,0 +1,25 @@
+#
+# Author:: Sean Carey (<densone@basho.com>)
+# Cookbook Name:: riak
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+default.riak.sysmon.process_limit = 30 
+default.riak.sysmon.port_limit = 2
+default.riak.sysmon.gc_ms_limit = 100
+default.riak.sysmon.heap_word_limit = 40111000
+default.riak.sysmon.busy_port = true  
+default.riak.sysmon.busy_dist_port = true  

data/chef/travis-cookbooks/ci_environment/riak/libraries/riak_template_helper.rb

@@ -0,0 +1,180 @@
+#
+# Author:: Benjamin Black (<b@b3k.us>) and Sean Cribbs (<sean@basho.com>)
+# Cookbook Name:: riak
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'delegate'
+module RiakTemplateHelper
+  class Tuple < DelegateClass(Array)
+    include RiakTemplateHelper
+    def to_s(depth=1)
+      "{" << map {|i| value_to_erlang(i,depth) }.join(", ") << "}"
+    end
+  end
+
+  def value_to_erlang(v, depth=1)
+    case v
+    when Hash
+      to_erlang_config(v, depth+1)
+    when String
+      "\"#{v}\""
+    when Array
+      "[" << v.map {|i| value_to_erlang(i) }.join(", ") << "]"
+    else
+      v.to_s
+    end
+  end
+
+  # Lifted from Ripple's Riak::TestServer
+  def to_erlang_config(hash, depth = 1)
+    padding = '    ' * depth
+    parent_padding = '    ' * (depth-1)
+
+    # Traverse through the hash in key sorted order so that we avoid
+    # any issues with the traversal order changing and forcing a Riak restart.
+    values = hash.keys.sort.map do |k|
+      v = hash[k]
+
+      if KEYLESS_ATTRIBUTES.include?(k)
+        #We make the assumption that all KEYLESS_ATTRIBUTES are arrays.
+        Tuple.new(v).to_s(depth)
+      else
+        "{#{k}, #{value_to_erlang(v, depth)}}"
+      end
+    end.join(",\n#{padding}")
+    "[\n#{padding}#{values}\n#{parent_padding}]"
+  end
+
+  #There are several configurations that are not key/value. They should be added to KEYLESS_ATTRIBUTES.
+  #A sample of this wold be the lager configuration.
+  #{"{{platform_log_dir}}/error.log", error, 10485760, "$D0", 5}
+  KEYLESS_ATTRIBUTES = ['lager_error_log','lager_console_log','default_user']
+
+  #Remove these configs. This will make sure package and erlang vms are not processed into the riak app.config.
+  RIAK_REMOVE_CONFIGS = ['package', 'erlang']
+
+  RIAK_TRANSLATE_CONFIGS = {
+    'api' => 'riak_api',
+    'core' => 'riak_core',
+    'kv' => 'riak_kv',
+    'search' => 'riak_search',
+    'err' => 'riak_err',
+    'sysmon' => 'riak_sysmon',
+    'control' => 'riak_control'
+  }
+
+
+  def prepare_app_config(riak)
+
+    #Each backend in multi-backend will be a keyless tuple, so add them to KEYLESS_ATTRIBUTES
+    #riak[:kv][:multi_backend].each_key { |k| KEYLESS_ATTRIBUTES.push(k) }
+
+
+    # Don't muck with the node attributes
+    riak = riak.to_hash
+
+    #remove net_dev_int from app.config
+    riak.delete('net_dev_int')
+
+    # Remove sections we don't care about
+    riak.reject! {|k,_| RIAK_REMOVE_CONFIGS.include? k }
+
+    # Rename sections
+    riak.dup.each do |k,v|
+      next if k.nil?
+      if RIAK_TRANSLATE_CONFIGS.include? k
+        riak[RIAK_TRANSLATE_CONFIGS[k]] = riak.delete(k)
+      end
+    end
+
+    # Only limit Erlang port range if limit_port_range is true
+    if riak['kernel'] && riak['kernel']['limit_port_range']
+      riak['kernel'].delete 'limit_port_range'
+    else
+      riak.delete 'kernel'
+    end
+
+    # Select the backend configuration
+    riak['riak_kv']['storage_backend'] = riak['riak_kv']['storage_backend'].to_sym
+    riak.delete('innostore') unless riak['riak_kv']['storage_backend']==:riak_kv_innostore_backend
+    riak.delete('eleveldb')  unless riak['riak_kv']['storage_backend']==:riak_kv_eleveldb_backend
+    riak.delete('bitcask')   unless riak['riak_kv']['storage_backend']==:riak_kv_bitcask_backend
+    riak['riak_kv'].delete('riak_kv_dets_backend_root') unless riak['riak_kv']['storage_backend']==:riak_kv_dets_backend
+    #This adds the multibackends in their own section of app.config for cs
+    if riak['riak_kv']['storage_backend']==:riak_cs_kv_multi_backend
+      node.riak.kv.multi_backend.each do |k,v|
+        riak[v[1].to_s.split('_')[2]] = v[2]
+      end
+    end
+
+    riak['riak_core']['default_bucket_props']['chash_keyfun'] = Tuple.new(riak['riak_core']['default_bucket_props']['chash_keyfun'].map {|i| i.to_sym }) if riak['riak_core']['default_bucket_props'] && riak['riak_core']['default_bucket_props']['chash_keyfun']
+
+    riak['riak_core']['default_bucket_props']['linkfun'] = Tuple.new(riak['riak_core']['default_bucket_props']['linkfun'].map {|i| i.to_sym }) if riak['riak_core']['default_bucket_props'] && riak['riak_core']['default_bucket_props']['linkfun']
+
+    allifs = lambda {|pair| pair[0] == "0.0.0.0" }
+    %w{http https}.each do |protocol|
+      if pairs = riak['riak_core'][protocol]
+        # If there's all-interfaces bindings ("0.0.0.0"), remove any
+        # specific interface bindings on the associated ports. Trying
+        # to bind twice will prevent riak from starting without much
+        # diagnostic information.
+        if pairs.any?(&allifs)
+          ports = pairs.select(&allifs).map {|pair| pair[1] }
+          pairs = pairs.delete_if {|(intf, port)| ports.include?(port) && intf != "0.0.0.0" }
+        end
+        riak['riak_core'][protocol] = pairs.uniq.map { |pair| Tuple.new(pair) }
+      end
+    end
+
+    # Return the sanitized config
+    riak
+  end
+
+  RIAK_VM_ARGS = {
+    "node_name" => "-name",
+    "cookie" => "-setcookie",
+    "heart" => "-heart",
+    "kernel_polling" => "+K",
+    "async_threads" => "+A",
+    "error_logger_warnings" => "+W",
+    "smp" => "-smp",
+    "env_vars" => "-env"
+  }
+
+  def prepare_vm_args(config)
+    config.keys.sort.map do |k|
+      v   = config[k]
+      key = RIAK_VM_ARGS[k.to_s]
+      case v
+      when false
+        nil
+      when Hash, Chef::Node::Attribute
+        # Mostly for env_vars
+        v.keys.sort.map { |ik| "#{key} #{ik} #{v[ik]}" }
+      else
+        "#{key} #{v}"
+      end
+    end.flatten.compact
+  end
+end
+
+class Chef::Resource::Template
+  include RiakTemplateHelper
+end
+
+class Erubis::Context
+  include RiakTemplateHelper
+end

data/chef/travis-cookbooks/ci_environment/riak/metadata.rb

@@ -0,0 +1,32 @@
+#
+# Author:: Benjamin Black (<b@b3k.us>) and Sean Cribbs (<sean@basho.com>)
+# Cookbook Name:: riak
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+maintainer        "Basho Technologies, Inc."
+maintainer_email  "riak@basho.com"
+license           "Apache 2.0"
+description       "Installs and configures Riak distributed data store"
+version           "1.2.0"
+recipe            "riak", "Installs Riak"
+recipe            "riak::autoconf", "Automatically configure nodes from chef-server information."
+recipe            "riak::innostore", "Install and configure the Innostore backend."
+recipe            "riak::iptables", "Automatically configure iptables rules for Riak."
+
+%w{ubuntu debian centos redhat suse fedora}.each do |os|
+  supports os
+end

data/chef/travis-cookbooks/ci_environment/riak/providers/cluster.rb

@@ -0,0 +1,80 @@
+#
+# Author:: Sean Cribbs (<sean@basho.com>)
+# Cookbook Name:: riak
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'timeout'
+require 'chef/mixin/shell_out'
+include Chef::Mixin::ShellOut
+include Timeout
+
+def load_current_resource
+  Chef::Application.fatal!("Can't join a Riak cluster without cluster_members.") if Chef::Config[:solo] && new_resource.cluster_members.blank?  
+  @current_resource ||= Chef::Resource::RiakCluster.new(new_resource.cluster_name)
+  current_resource.ring_ready(ringready)
+  Chef::Application.fatal!("Can't join a Riak cluster if the local node is not running.") unless current_resource.ring_ready[:running]
+  current_resource.joined(!current_resource.ring_ready[:ready] || current_resource.ring_ready[:members].size > 1)
+  if new_resource.cluster_members
+    current_resource.cluster_members(new_resource.cluster_members)
+  else
+    current_resource.cluster_members([])
+    search(:node, "riak_core_cluster_name:#{new_resource.cluster_name}") do |n|
+      current_resource.cluster_members << n.riak.erlang.node_name
+    end
+  end
+end
+
+action :join do
+  if !current_resource.joined && peer = random_cluster_member
+    execute "Joining node #{new_resource.node_name} to Riak cluster #{new_resource.cluster_name}" do
+      command "riak-admin join #{peer}"
+      path [new_resource.riak_admin_path]
+    end
+    new_resource.updated_by_last_action(true)
+  end
+  action_wait_for_ringready
+end
+
+action :wait_for_ringready do
+  begin
+    timeout(new_resource.timeout) do
+      wait_for_ring
+    end
+  rescue Timeout::Error
+    Chef::Application.fatal! "Riak cluster ring did not converge within #{new_resource.timeout} seconds!"
+  end
+end
+
+private
+
+def ringready
+  cmd = shell_out("#{new_resource.riak_admin_path}/riak-admin ringready")
+  {
+    :ready => cmd.stdout =~ /TRUE/,
+    :members => cmd.stdout.scan(/'([^',]+)'/).flatten,
+    :running => cmd.stdout !~ /Node is not running/
+  }
+end
+
+def wait_for_ring
+  sleep 1 until ringready[:ready]
+end
+
+def random_cluster_member
+  tmp = current_resource.cluster_members.reject {|n| n == new_resource.node_name }
+  tmp[ rand(tmp.size) ]
+end

data/chef/travis-cookbooks/ci_environment/riak/recipes/autoconf.rb

@@ -0,0 +1,47 @@
+#
+# Author:: Benjamin Black (<b@b3k.us>) and Sean Cribbs (<sean@basho.com>)
+# Cookbook Name:: riak
+# Recipe:: autoconf
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "riak::default"
+
+if node[:riak][:package][:type] == "binary"
+  bin_path = "/usr/sbin"
+else
+  bin_path = "#{node[:riak][:package][:prefix]}/riak/bin"
+end
+
+# Riak's packaged init.d script doesn't work. We'll use the bin script
+# instead.
+bash "Start riak and wait for riak_kv to be available" do
+  code <<-SCRIPT
+#{bin_path}/riak start
+#{bin_path}/riak-admin wait-for-service riak_kv #{node[:riak][:erlang][:node_name]}
+SCRIPT
+  timeout 45
+end
+
+# Need to make sure the riak attributes are saved to the server so we
+# don't blow up in the riak_cluster provider on not having a node name.
+node.save
+
+riak_cluster node[:riak][:core][:cluster_name] do
+  node_name node[:riak][:erlang][:node_name]
+  action :join
+  riak_admin_path bin_path
+end

data/chef/travis-cookbooks/ci_environment/riak/recipes/config.rb

@@ -0,0 +1,70 @@
+#
+# Author:: Sean Cribbs (<sean@basho.com>)
+# Cookbook Name:: riak
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# package.rb
+if (node.riak.package.type.eql?("source"))
+  node.default.riak.package.prefix = "/usr/local"
+  node.default.riak.package.config_dir = node.riak.package.prefix + "/riak/etc"
+end
+
+# erlang.rb
+node.default.riak.erlang.node_name = "riak@127.0.0.1"
+
+# core.rb
+# Make sure the bare minimums are set so the cluster works
+if node.riak.core.to_hash['default_bucket_props'].is_a?(Mash)
+  node.default.riak.core.default_bucket_props.n_val = 3
+  node.default.riak.core.default_bucket_props.chash_keyfun = [:riak_core_util, :chash_std_keyfun]
+end
+
+# kernel.rb
+unless node.riak.kernel.limit_port_range
+  node.riak.kernel.delete(:inet_dist_listen_min)
+  node.riak.kernel.delete(:inet_dist_listen_max)
+end
+
+
+# kv.rb
+node.set.riak.kv.storage_backend = (node.riak.kv.storage_backend).to_s.to_sym
+
+# sasl.rb
+node.set.riak.sasl.errlog_type = (node.riak.sasl.errlog_type).to_s.to_sym
+
+
+case node.riak.kv.storage_backend
+when :riak_kv_bitcask_backend # bitcask.rb
+  node.set['riak'].delete(:eleveldb)
+  node.set['riak'].delete(:innostore)
+  node.set['riak'].kv.delete(:riak_kv_dets_backend_root)
+  unless (node.riak.bitcask).to_hash["sync_strategy"].is_a?(Mash)
+    node.riak.bitcask.sync_strategy = (node.riak.bitcask.sync_strategy).to_s.to_sym
+  end
+when :riak_kv_eleveldb_backend # eleveldb.rb
+  node.set['riak'].delete(:bitcask)
+  node.set['riak'].delete(:innostore)
+  node.set['riak'].kv.delete(:riak_kv_dets_backend_root)
+when :riak_kv_dets_backend # dets.rb
+  node.set['riak'].delete(:bitcask)
+  node.set['riak'].delete(:eleveldb)
+  node.set['riak'].delete(:innostore)
+when # innostore.rb
+  node.set['riak'].delete(:bitcask)
+  node.set['riak'].delete(:eleveldb)
+  node.set['riak'].kv.delete(:riak_kv_dets_backend_root)
+end

data/chef/travis-cookbooks/ci_environment/riak/recipes/default.rb

@@ -0,0 +1,160 @@
+#
+# Author:: Benjamin Black (<b@b3k.us>) and Sean Cribbs (<sean@basho.com>)
+# Cookbook Name:: riak
+# Recipe:: default
+#
+# Copyright (c) 2011 Basho Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "riak::config"
+
+case node.platform
+when "ubuntu" then
+  include_recipe "libssl::098"
+end
+
+if node[:riak][:package][:url]
+  package_uri = node[:riak][:package][:url]
+  package_file = package_uri.split("/").last
+else
+  version_str = "#{node[:riak][:package][:version][:major]}.#{node[:riak][:package][:version][:minor]}"
+  base_uri = "http://s3.amazonaws.com/downloads.basho.com/riak/#{version_str}/#{version_str}.#{node[:riak][:package][:version][:incremental]}"
+  base_filename = "riak-#{version_str}.#{node[:riak][:package][:version][:incremental]}"
+
+  case node[:platform]
+  when "debian","ubuntu"
+    machines = {"x86_64" => "amd64", "i386" => "i386", "i686" => "i386"}
+    base_uri = "#{base_uri}/#{node[:platform]}/#{node[:lsb][:codename]}/"
+  when "redhat","centos","scientific","fedora","suse"
+    machines = {"x86_64" => "x86_64", "i386" => "i386", "i686" => "i686"}
+    base_uri = "#{base_uri}/rhel/#{node[:platform_version].to_i}/"
+  when "fedora"
+    machines = {"x86_64" => "x86_64", "i386" => "i386", "i686" => "i686"}
+    base_uri = "#{base_uri}/#{node[:platform]}/#{node[:platform_version].to_i}/"
+  end
+  package_file =  case node[:riak][:package][:type]
+                  when "binary"
+                    case node[:platform]
+                    when "debian","ubuntu"
+                      "#{base_filename.gsub(/\-/, '_')}-#{node[:riak][:package][:version][:build]}_#{machines[node[:kernel][:machine]]}.deb"
+                    when "centos","redhat","suse"
+                      if node[:platform_version].to_i == 6
+                        "#{base_filename}-#{node[:riak][:package][:version][:build]}.el6.#{machines[node[:kernel][:machine]]}.rpm"
+                      else
+                        "#{base_filename}-#{node[:riak][:package][:version][:build]}.el5.#{machines[node[:kernel][:machine]]}.rpm"
+                      end
+                    when "fedora"
+                      "#{base_filename}-#{node[:riak][:package][:version][:build]}.fc13.#{node[:kernel][:machine]}.rpm"
+                    end
+                  when "source"
+                    "#{base_filename}.tar.gz"
+                  end
+  package_uri = base_uri + package_file
+end
+
+package_name = package_file.split("[-_]\d+\.").first
+
+#if %w{debian ubuntu}.include?(node[:platform])
+#  include_recipe "riak::iptables"
+#end
+
+group "riak"
+
+user "riak" do
+  gid "riak"
+  shell "/bin/bash"
+  home "/var/lib/riak"
+  system true
+end
+
+directory "/tmp/riak_pkg" do
+  owner "root"
+  mode 0755
+  action :create
+end
+
+remote_file "/tmp/riak_pkg/#{package_file}" do
+  source package_uri
+  owner "root"
+  mode 0644
+  checksum node[:riak][:package][:source_checksum]
+  not_if { File.exists?("/tmp/riak_pkg/#{package_file}") }
+end
+
+case node[:riak][:package][:type]
+when "binary"
+  package package_name do
+    source "/tmp/riak_pkg/#{package_file}"
+    action :install
+    options case node[:platform]
+            when "debian","ubuntu"
+              "--force-confnew"
+            end
+    provider value_for_platform(
+      [ "ubuntu", "debian" ] => {"default" => Chef::Provider::Package::Dpkg},
+      [ "redhat", "centos", "fedora", "suse" ] => {"default" => Chef::Provider::Package::Rpm}
+    )
+  end
+when "source"
+  execute "riak-src-unpack" do
+    cwd "/tmp/riak_pkg"
+    command "tar xvfz #{package_file}"
+  end
+
+  execute "riak-src-build" do
+    cwd "/tmp/riak_pkg/#{base_filename}"
+    command "make clean all rel"
+  end
+
+  execute "riak-src-install" do
+    command "mv /tmp/riak_pkg/#{base_filename}/rel/riak #{node[:riak][:package][:prefix]}"
+    not_if { File.directory?("#{node[:riak][:package][:prefix]}/riak") }
+  end
+end
+
+case node[:riak][:kv][:storage_backend]
+when :riak_kv_innostore_backend
+  include_recipe "riak::innostore"
+end
+
+directory node[:riak][:package][:config_dir] do
+  owner "root"
+  mode "0755"
+  action :create
+end
+
+template "#{node[:riak][:package][:config_dir]}/app.config" do
+  source "app.config.erb"
+  owner "root"
+  mode 0644
+end
+
+template "#{node[:riak][:package][:config_dir]}/vm.args" do
+  variables :switches => prepare_vm_args(node[:riak][:erlang].to_hash)
+  source "vm.args.erb"
+  owner "root"
+  mode 0644
+end
+
+if node[:riak][:package][:type].eql?("binary")
+  service "riak" do
+    supports :start => true, :stop => true, :restart => true
+    if node.riak.service.enabled
+      action [:enable]
+    else
+      action [:disable]
+    end
+  end
+end