dust-deploy 0.6.1 → 0.6.2

data/changelog.md

@@ -1,6 +1,21 @@
 Changelog
 =============
 
+0.6.2
+------------
+
+-  adds redis recipe, you can now maintain your redis configurations with dust as well:
+
+    recipes:
+      redis:
+        port: 6379
+        daemonize: yes
+
+-  fixes hash_check recipe, now works with centos-like machines as well
+-  improves mysql recipe: now sets shm sysctls as well (like the postgresql recipe does)
+-  small improvements to automatic innodb tuning
+
+
 0.6.1
 ------------
 
@@ -80,6 +95,7 @@ Changelog
 ------------
 
 sshd recipe
+
 -  default PrintMotd to false on apt systems (will be displayed 2 times otherwise)
 -  no and yes can be specified in config file, without getting converted to booleans automatically 
 

data/lib/dust/recipes/hash_check.rb

@@ -2,16 +2,9 @@ class HashCheck < Recipe
   
   desc 'hash_check:deploy', 'checks /etc/shadow for weak hashes'
   def deploy
-    # mkpasswd is in the package 'whois' resp. 'expect'
-    @node.install_package 'whois' if @node.uses_apt?
-    @node.install_package 'expect' if @node.uses_rpm?
-
     # those keys indicate that no password is set, or login is disabled
     keys = [ '*', '!', '!!', '', 'LK', 'NP' ]
 
-    # mapping the magic numbers to the actual hash algorithms
-    algorithms = { '1' => 'md5', '2' => 'blowfish', '5' => 'sha-256', '6' => 'sha-512' }
-
     weak_passwords = File.open "#{@template_path}/weak_passwords", 'r'
     shadow = @node.exec('cat /etc/shadow')[:stdout]
 
@@ -23,17 +16,22 @@ class HashCheck < Recipe
       user, hash = line.split(':')[0..1]
       next if keys.include? hash
       method, salt = hash.split('$')[1..2]
-  
+
       weak_passwords.each_line do |password|
         password.chomp!
 
-        # generate the hash for this password, according to salt and method
-        weak_hash = @node.exec("mkpasswd -m #{algorithms[method.to_s]} -S '#{salt}' '#{password}'")[:stdout]
-        weak_hash.chomp!
+        # python was imho the best solution to generate /etc/shadow hashes.
+        # mkpasswd doesn't work on centos-like machines :/
+        # and python is more likely installed than ruby
+        ret = @node.exec("python -c \"import crypt; print crypt.crypt('#{password}', '\\$#{method}\\$#{salt}\\$')\"")
 
-        if weak_hash == hash
+        unless ret[:exit_code] == 0
+          ::Dust.print_failed 'error during hash creation (is python installed?)'
+          return false
+        end
+        if hash == ret[:stdout].chomp
           ::Dust.print_failed "user #{user} has a weak password! (#{password})", :indent => 2
-          found_weak= true
+          found_weak = true
         end
       end
     end

data/lib/dust/recipes/mysql.rb

@@ -9,12 +9,14 @@ class Mysql < Recipe
     ::Dust.print_msg "configuring mysql\n"
     ::Dust.print_ok "listen on #{@config['mysqld']['bind-address']}:#{@config['mysqld']['port']}", :indent => 2
 
-    @config['mysqld']['innodb_buffer_pool_size'] = get_innodb_buffer_pool_size
+    @config['mysqld']['innodb_buffer_pool_size'] ||= get_innodb_buffer_pool_size
     ::Dust.print_ok "set innodb buffer pool to '#{@config['mysqld']['innodb_buffer_pool_size']}'", :indent => 2
 
     @node.write '/etc/mysql/my.cnf', generate_my_cnf
     @node.chmod '644', '/etc/mysql/my.cnf'
-
+        
+    configure_sysctl
+    
     @node.restart_service 'mysql' if options.restart?
     @node.reload_service 'mysql' if options.reload?
   end
@@ -53,7 +55,9 @@ class Mysql < Recipe
         'max_binlog_size' => '100M',
         'innodb_file_per_table' => 1,
         'innodb_thread_concurrency' => 0,
-        'innodb_flush_log_at_trx_commit' => 1
+        'innodb_flush_log_at_trx_commit' => 1,
+        'innodb_additional_mem_pool_size' => '16M',
+        'innodb_log_buffer_size' => '4M'
       },
       'mysqldump' => {
         'quick' => true,
@@ -77,12 +81,12 @@ class Mysql < Recipe
       # get system memory (in kb)
       system_mem = ::Dust.convert_size @node['memorysize']
   
-      # allocate 70% of the available ram to mysql
-      buffer_pool = (system_mem * 0.70).to_i / 1024
-     
+      # allocate 80% of the available ram to mysql
+      buffer_pool = (system_mem * 0.8).to_i
+      
       ::Dust.print_ok
-      "#{buffer_pool}M"
-    end
+      "#{buffer_pool / 1024}M"
+    end    
   end
   
   def generate_my_cnf
@@ -97,5 +101,53 @@ class Mysql < Recipe
     my_cnf.concat "!includedir /etc/mysql/conf.d/\n"
     my_cnf
   end
+  
+  # increase shm memory  
+  def configure_sysctl
+    if @node.uses_apt?
+      ::Dust.print_msg "setting mysql sysctl keys\n"
+      @node.collect_facts :quiet => true
+     
+      # make sure system allows more than innodb_buffer_pool_size of memory ram to be allocated
+      # shmmax = (convert_mysql_size(@config['mysqld']['innodb_buffer_pool_size']) * 1.1).to_i # TODO: 1.1?
+
+      # get pagesize
+      pagesize = @node.exec('getconf PAGESIZE')[:stdout].to_i || 4096
+
+      # use half of system memory for shmmax
+      shmmax = ::Dust.convert_size(@node['memorysize']) * 1024 / 2
+      shmall = shmmax / pagesize
+      
+      ::Dust.print_msg "setting shmmax to: #{shmmax}", :indent => 2
+      ::Dust.print_result @node.exec("sysctl -w kernel.shmmax=#{shmmax}")[:exit_code]
+      ::Dust.print_msg "setting shmall to: #{shmall}", :indent => 2
+      ::Dust.print_result @node.exec("sysctl -w kernel.shmall=#{shmall}")[:exit_code]
+      ::Dust.print_msg 'setting swappiness to 0', :indent => 2
+      ::Dust.print_result @node.exec('sysctl -w vm.swappiness=0')[:exit_code]
+
+      file = ''
+      file += "kernel.shmmax=#{shmmax}\n"
+      file += "kernel.shmall=#{shmall}\n"
+      file += "vm.swappiness=0\n" # rather shrink cache then use swap as filesystem cache
+      
+      @node.write "/etc/sysctl.d/30-mysql-shm.conf", file
+      
+      else
+      ::Dust.print_warning 'sysctl configuration not supported for your os'
+    end
+  end
+  
+  def convert_mysql_size s
+    case s[-1].chr
+      when 'K'
+      return (s[0..-2].to_f * 1024).to_i
+      when 'M'
+      return (s[0..-2].to_f * 1024 * 1024).to_i
+      when 'G'
+      return (s[0..-2].to_f * 1024 * 1024 * 1024).to_i
+      else
+      return s.to_i
+    end
+  end
 end
 

data/lib/dust/recipes/postgres.rb

@@ -93,9 +93,12 @@ class Postgres < Recipe
       ::Dust.print_msg "setting postgres sysctl keys\n"
       @node.collect_facts :quiet => true
 
+      # get pagesize
+      pagesize = @node.exec('getconf PAGESIZE')[:stdout] || 4096
+
       # use half of system memory for shmmax
       shmmax = ::Dust.convert_size(@node['memorysize']) * 1024 / 2
-      shmall = shmmax / 4096 # shmmax/pagesize (pagesize = 4096)
+      shmall = shmmax / pagesize
 
       ::Dust.print_msg "setting shmmax to: #{shmmax}", :indent => 2
       ::Dust.print_result @node.exec("sysctl -w kernel.shmmax=#{shmmax}")[:exit_code]

data/lib/dust/recipes/redis.rb

@@ -0,0 +1,91 @@
+class Redis < Recipe
+  desc 'redis:deploy', 'installs and configures redis key-value store'
+  def deploy
+    @node.install_package 'redis-server'
+    @node.write '/etc/redis/redis.conf', generate_redis_conf
+    configure_sysctl
+    @node.restart_service 'redis-server' if @options.restart
+  end
+
+  desc 'redis:status', 'displays redis-cli info'
+  def status
+    return false unless @node.package_installed? 'redis-server'
+    puts @node.exec('redis-cli info')[:stdout]
+  end
+  
+  
+  private
+  
+  # default configuration variables for ubuntu
+  # if you use a different os, you may adapt these
+  # listens on all interfaces per default
+  def default_config
+    { 'daemonize' => 'yes',
+      'port' => 6379,
+      'timeout' => 300,
+      'loglevel' => 'notice',
+      'databases' => 16,
+      'save' => [ '900 1', '300 10', '60 10000' ],
+      'rdbcompression' => 'yes',
+      'dbfilename' => 'dump.rdb',
+      'slave-serve-stale-data' => 'yes',
+      'appendonly' => 'no',
+      'appendfsync' => 'everysec',
+      'no-appendfsync-on-rewrite' => 'no',
+      'vm-enabled' => 'no',
+      'vm-max-memory' => 0,
+      'vm-page-size' => 32,
+      'vm-pages' => 134217728,
+      'vm-max-threads' => 4,
+      'hash-max-zipmap-entries' => 512,
+      'hash-max-zipmap-value' => 64,
+      'list-max-ziplist-entries' => 512,
+      'list-max-ziplist-value' => 64,
+      'set-max-intset-entries' => 512,
+      'activerehashing' => 'yes',
+
+      # os specific settings
+      'dir' => '/var/lib/redis',
+      'pidfile' => '/var/run/redis.pid',
+      'logfile' => '/var/log/redis/redis-server.log',
+      'vm-swap-file' => '/var/lib/redis/redis.swap'
+    }
+  end
+  
+  def generate_redis_conf
+    @config.boolean_to_string!
+    @config = default_config.merge @config
+    
+    redis_conf = ''
+    @config.each do |key, value|
+      if value.is_a? Array
+        value.each { |v| redis_conf.concat "#{key} #{v}\n" }
+      else
+        redis_conf.concat "#{key} #{value}\n"
+      end
+    end
+    
+    redis_conf
+  end
+  
+  # redis complains if vm.overcommit_memory != 1
+  def configure_sysctl
+    if @node.uses_apt?
+      ::Dust.print_msg "setting redis sysctl keys\n"
+      
+      ::Dust.print_msg 'setting overcommit memory to 1', :indent => 2
+      ::Dust.print_result @node.exec('sysctl -w vm.overcommit_memory=1')[:exit_code]
+      ::Dust.print_msg 'setting swappiness to 0', :indent => 2
+      ::Dust.print_result @node.exec('sysctl -w vm.swappiness=0')[:exit_code]
+      
+      file = ''
+      file += "vm.overcommit_memory=1\n"
+      file += "vm.swappiness=0\n"
+      
+      @node.write "/etc/sysctl.d/30-redis.conf", file
+      
+    else
+      ::Dust.print_warning 'sysctl configuration not supported for your os'
+    end
+  end
+end

data/lib/dust/version.rb

@@ -1,3 +1,3 @@
 module Dust
-  VERSION = "0.6.1"
+  VERSION = "0.6.2"
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 6
-  - 1
-  version: 0.6.1
+  - 2
+  version: 0.6.2
 platform: ruby
 authors: 
 - kris kechagia
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-31 00:00:00 +01:00
+date: 2012-02-02 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -139,6 +139,7 @@ files:
 - lib/dust/recipes/packages.rb
 - lib/dust/recipes/postgres.rb
 - lib/dust/recipes/rc_local.rb
+- lib/dust/recipes/redis.rb
 - lib/dust/recipes/remove_packages.rb
 - lib/dust/recipes/repositories.rb
 - lib/dust/recipes/resolv_conf.rb