server_maint 0.0.3 → 0.0.4

data/lib/cookbooks/database/recipes/ebs_volume.rb

@@ -0,0 +1,204 @@
+#
+# Author:: Joshua Timberman (<joshua@opscode.com>)
+# Author:: AJ Christensen (<aj@opscode.com>)
+# Cookbook Name:: database
+# Recipe:: ebs_volume
+#
+# Copyright 2009-2010, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if node[:ec2]
+  include_recipe "aws"
+  include_recipe "xfs"
+
+  begin
+    aws = Chef::DataBagItem.load(:aws, :main)
+    Chef::Log.info("Loaded AWS information from DataBagItem aws[#{aws['id']}]")
+  rescue
+    Chef::Log.fatal("Could not find the 'main' item in the 'aws' data bag")
+    raise
+  end
+
+  ebs_vol_dev = node['mysql']['ebs_vol_dev']
+  if (platform?("ubuntu") && node['platform_version'].to_f >= 11.04)
+    ebs_vol_dev_mount =  ebs_vol_dev.sub(/^\/dev\/sd/, "/dev/xvd")
+  else
+    ebs_vol_dev_mount = ebs_vol_dev
+  end
+  ebs_vol_id = String.new
+  db_type = String.new
+  db_role = String.new
+  master_role = String.new
+  slave_role = String.new
+  root_pw = String.new
+  snapshots_to_keep = String.new
+  snapshot_cron_schedule = "00 * * * *" # default to hourly snapshots
+
+  search(:apps) do |app|
+    if (app["database_master_role"] & node.run_list.roles).length == 1 || (app["database_slave_role"] & node.run_list.roles).length == 1
+      master_role = app["database_master_role"] & node.run_list.roles
+      slave_role = app["database_slave_role"] & node.run_list.roles
+      root_pw = app["mysql_root_password"][node.chef_environment]
+      snapshots_to_keep = app["snapshots_to_keep"][node.chef_environment]
+      snapshot_cron_schedule = app["snapshot_cron_schedule"][node.chef_environment] if app["snapshot_cron_schedule"] && app["snapshot_cron_schedule"][node.chef_environment]
+
+      if (master_role & node.run_list.roles).length == 1
+        db_type = "master"
+        db_role = RUBY_VERSION.to_f <= 1.8 ? master_role : master_role.join
+      elsif (slave_role & node.run_list.roles).length == 1
+        db_type = "slave"
+        db_role = RUBY_VERSION.to_f <= 1.8 ? slave_role : slave_role.join
+      end
+
+      Chef::Log.info "database::ebs_volume - db_role: #{db_role} db_type: #{db_type}"
+    end
+  end
+
+  begin
+    ebs_info = Chef::DataBagItem.load(:aws, "ebs_#{db_role}_#{node.chef_environment}")
+    Chef::Log.info("Loaded #{ebs_info['volume_id']} from DataBagItem aws[#{ebs_info['id']}]")
+  rescue
+    Chef::Log.warn("Could not find the 'ebs_#{db_role}_#{node.chef_environment}' item in the 'aws' data bag")
+    ebs_info = Hash.new
+  end
+
+  begin
+    master_info = Chef::DataBagItem.load(:aws, "ebs_#{master_role}_#{node.chef_environment}")
+    Chef::Log.info "Loaded #{master_info['volume_id']} from DataBagItem aws[#{master_info['id']}]"
+  rescue
+    Chef::Application.fatal! "Could not load replication masters snapshot details", -41 if db_type == "slave"
+  end
+
+  ruby_block "store_#{db_role}_#{node.chef_environment}_volid" do
+    block do
+      ebs_vol_id = node[:aws][:ebs_volume]["#{db_role}_#{node.chef_environment}"][:volume_id]
+
+      unless ebs_info['volume_id']
+        item = {
+          "id" => "ebs_#{db_role}_#{node.chef_environment}",
+          "volume_id" => ebs_vol_id
+        }
+        Chef::Log.info "Storing volume_id #{item.inspect}"
+        databag_item = Chef::DataBagItem.new
+        databag_item.data_bag("aws")
+        databag_item.raw_data = item
+        databag_item.save
+        Chef::Log.info("Created #{item['id']} in #{databag_item.data_bag}")
+      end
+    end
+    action :nothing
+  end
+
+  aws_ebs_volume "#{db_role}_#{node.chef_environment}" do
+    aws_access_key aws['aws_access_key_id']
+    aws_secret_access_key aws['aws_secret_access_key']
+    size 50
+    device ebs_vol_dev
+    snapshots_to_keep snapshots_to_keep
+    case db_type
+    when "master"
+      if ebs_info['volume_id'] && ebs_info['volume_id'] =~ /vol/
+        volume_id ebs_info['volume_id']
+        action :attach
+      elsif ebs_info['volume_id'] && ebs_info['volume_id'] =~ /snap/
+        snapshot_id ebs_info['volume_id']
+        action [ :create, :attach ]
+      else
+        action [ :create, :attach ]
+      end
+      notifies :create, resources(:ruby_block => "store_#{db_role}_#{node.chef_environment}_volid")
+    when "slave"
+      if master_info['volume_id']
+        snapshot_id master_info['volume_id']
+        action [:create, :attach]
+      else
+        Chef::Log.warn("Couldn't detect snapshot ID.")
+        action :nothing
+      end
+    end
+    provider "aws_ebs_volume"
+  end
+
+  if db_type == "master"
+    Chef::Log.info "Setting up templates for chef-solo snapshots"
+    template "/etc/chef/chef-solo-database-snapshot.rb" do
+      source "chef-solo-database-snapshot.rb.erb"
+      variables :cookbook_path => Chef::Config[:cookbook_path]
+      owner "root"
+      group "root"
+      mode 0600
+    end
+
+    template "/etc/chef/chef-solo-database-snapshot.json" do
+      source "chef-solo-database-snapshot.json.erb"
+      variables(
+        :output => {
+          'db_snapshot' => {
+            'ebs_vol_dev' => node.mysql.ec2_path,
+            'db_role' => db_role,
+            'app_environment' => node.chef_environment,
+            'username' => 'root',
+            'password' => root_pw,
+            'aws_access_key_id' => aws['aws_access_key_id'],
+            'aws_secret_access_key' => aws['aws_secret_access_key'],
+            'snapshots_to_keep' => snapshots_to_keep,
+            'volume_id' => ebs_info['volume_id']
+          },
+          'run_list' => [
+            "recipe[database::snapshot]"
+          ]
+        }
+      )
+      owner "root"
+      group "root"
+      mode 0600
+    end
+
+    template "/etc/cron.d/chef-solo-database-snapshot" do
+      source "chef-solo-database-snapshot.cron.erb"
+      variables(
+        :json_attribs => "/etc/chef/chef-solo-database-snapshot.json",
+        :config_file => "/etc/chef/chef-solo-database-snapshot.rb",
+        :schedule => snapshot_cron_schedule
+      )
+      owner "root"
+      group "root"
+      mode 0600
+    end
+  end
+
+    execute "mkfs.xfs #{ebs_vol_dev_mount}" do
+      only_if "xfs_admin -l #{ebs_vol_dev_mount} 2>&1 | grep -qx 'xfs_admin: #{ebs_vol_dev_mount} is not a valid XFS filesystem (unexpected SB magic number 0x00000000)'"
+    end
+
+  %w{ec2_path data_dir}.each do |dir|
+    directory node['mysql'][dir] do
+      mode 0755
+    end
+  end
+
+  mount node['mysql']['ec2_path'] do
+    device ebs_vol_dev_mount
+    fstype "xfs"
+    action :mount
+  end
+
+  mount node['mysql']['data_dir'] do
+    device node['mysql']['ec2_path']
+    fstype "none"
+    options "bind,rw"
+    action :mount
+  end
+end

data/lib/cookbooks/database/recipes/master.rb

@@ -0,0 +1,78 @@
+#
+# Author:: Joshua Timberman (<joshua@opscode.com>)
+# Cookbook Name:: database
+# Recipe:: master
+#
+# Copyright 2009-2010, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is potentially destructive to the nodes mysql password attributes, since
+# we iterate over all the app databags. If this database server provides
+# databases for multiple applications, the last app found in the databags
+# will win out, so make sure the databags have the same passwords set for
+# the root, repl, and debian-sys-maint users.
+#
+
+db_info = Hash.new
+root_pw = String.new
+
+search(:apps) do |app|
+  (app['database_master_role'] & node.run_list.roles).each do |dbm_role|
+    %w{ root repl debian }.each do |user|
+      user_pw = app["mysql_#{user}_password"]
+      if !user_pw.nil? and user_pw[node.chef_environment]
+        Chef::Log.debug("Saving password for #{user} as node attribute node['mysql']['server_#{user}_password'")
+        node.set['mysql']["server_#{user}_password"] = user_pw[node.chef_environment]
+        node.save
+      else
+        log "A password for MySQL user #{user} was not found in DataBag 'apps' item '#{app["id"]}' for environment ' for #{node.chef_environment}'." do
+          level :warn
+        end
+        log "A random password will be generated by the mysql cookbook and added as 'node.mysql.server_#{user}_password'. Edit the DataBag item to ensure it is set correctly on new nodes" do
+          level :warn
+        end
+      end
+    end
+    app['databases'].each do |env,db|
+      db_info[env] = db
+    end
+  end
+end
+
+include_recipe "mysql::server"
+
+connection_info = {:host => "localhost", :username => 'root', :password => node['mysql']['server_root_password']}
+
+search(:apps) do |app|
+  (app['database_master_role'] & node.run_list.roles).each do |dbm_role|
+    app['databases'].each do |env,db|
+      if env =~ /#{node.chef_environment}/
+        mysql_database "create #{db['database']}" do
+          database_name db['database']
+          connection connection_info
+          action :create
+        end
+        %W{ % #{node['fqdn']} localhost }.each do |h|
+          mysql_database_user db['username'] do
+            connection connection_info
+            password db['password']
+            database_name db['database']
+            host h
+            action :grant
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cookbooks/database/recipes/mysql.rb

@@ -0,0 +1,20 @@
+#
+# Author:: Jesse Howarth (<him@jessehowarth.com>)
+#
+# Copyright:: Copyright (c) 2012, Opscode, Inc. (<legal@opscode.com>)
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "mysql::ruby"

data/lib/cookbooks/database/recipes/postgresql.rb

@@ -0,0 +1,20 @@
+#
+# Author:: Jesse Howarth (<him@jessehowarth.com>)
+#
+# Copyright:: Copyright (c) 2012, Opscode, Inc. (<legal@opscode.com>)
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe "postgresql::ruby"

data/lib/cookbooks/database/recipes/snapshot.rb

@@ -0,0 +1,62 @@
+#
+# Author:: AJ Christensen (<aj@opscode.com>)
+# Cookbook Name:: database
+# Recipe:: snapshot
+#
+# Copyright 2009-2010, Opscode, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+include_recipe "aws"
+include_recipe "xfs"
+
+%w{ebs_vol_dev db_role app_environment username password aws_access_key_id aws_secret_access_key snapshots_to_keep volume_id}.collect do |key|
+  Chef::Application.fatal!("Required db_snapshot configuration #{key} not found.", -47) unless node.db_snapshot.has_key? key
+end
+
+connection_info = {:host => "localhost", :username => node.db_snapshot.username, :password => node.db_snapshot.password}
+
+mysql_database "locking tables for #{node.db_snapshot.app_environment}" do
+  connection connection_info
+  sql "flush tables with read lock"
+  action :query
+end
+
+execute "xfs freeze" do
+  command "xfs_freeze -f #{node.db_snapshot.ebs_vol_dev}"
+end
+
+aws_ebs_volume "#{node.db_snapshot.db_role.first}_#{node.db_snapshot.app_environment}" do
+  aws_access_key node.db_snapshot.aws_access_key_id
+  aws_secret_access_key node.db_snapshot.aws_secret_access_key
+  size 50
+  device node.db_snapshot.ebs_vol_dev
+  snapshots_to_keep node.db_snapshot.snapshots_to_keep
+  action :snapshot
+  volume_id node.db_snapshot.volume_id
+  ignore_failure true # if this fails, continue to unfreeze and unlock
+end
+
+execute "xfs unfreeze" do
+  command "xfs_freeze -u #{node.db_snapshot.ebs_vol_dev}"
+end
+
+mysql_database "unflushing tables for #{node.db_snapshot.app_environment}" do
+  connection connection_info
+  sql "unlock tables"
+  action :query
+end
+
+aws_ebs_volume "#{node.db_snapshot.db_role.first}_#{node.db_snapshot.app_environment}" do
+  action :prune
+end

data/lib/cookbooks/database/templates/default/app_grants.sql.erb

@@ -0,0 +1,8 @@
+# Generated by Chef. Local modifications will be overwritten.
+<% @db_info.each do |env,db| -%>
+# Privileges for databases in <%= env %>
+GRANT ALL ON <%= db['database'] %>.* TO '<%= db['username'] %>'@'localhost' IDENTIFIED BY '<%= db['password'] %>';
+GRANT ALL ON <%= db['database'] %>.* TO '<%= db['username'] %>'@'<%= node['fqdn'] %>' IDENTIFIED BY '<%= db['password'] %>';
+GRANT ALL ON <%= db['database'] %>.* TO '<%= db['username'] %>'@'%' IDENTIFIED BY '<%= db['password'] %>';
+<% end -%>
+flush privileges;

data/lib/cookbooks/database/templates/default/aws_config.erb

@@ -0,0 +1,3 @@
+AWS_ACCESS_KEY_ID=<%= @access_key %>
+AWS_SECRET_ACCESS_KEY=<%= @secret_key %>
+BUCKET_BASE_NAME=db-backups

data/lib/cookbooks/database/templates/default/chef-solo-database-snapshot.cron.erb

@@ -0,0 +1,6 @@
+# Managed by Chef
+# m h  dom mon dow   command
+# Keep 1 day of hourly snapshots
+PATH=/usr/sbin:/usr/bin:/sbin:/bin
+<% cs = "chef-solo -j #{@json_attribs} -c #{@config_file}" %>
+<%= @schedule %> root <%= cs %>

data/lib/cookbooks/database/templates/default/chef-solo-database-snapshot.json.erb

@@ -0,0 +1 @@
+<%= require 'json'; JSON.pretty_generate(@output) %>

data/lib/cookbooks/database/templates/default/chef-solo-database-snapshot.rb.erb

@@ -0,0 +1,6 @@
+executable_path ENV['PATH'] ? ENV['PATH'].split(File::PATH_SEPARATOR) : []
+<% if @cookbook_path.is_a? Array %>
+  cookbook_path [ <%= @cookbook_path.collect { |cb| "\"#{cb}\""}.join(", ") -%> ]
+<% else %>
+  cookbook_path "<%= @cookbook_path -%>"
+<% end %>

data/lib/cookbooks/database/templates/default/ebs-backup-cron.erb

@@ -0,0 +1,2 @@
+# Chef Name: ebs_db_backup
+15 0 * * * root /usr/local/bin/db-backup.sh

data/lib/cookbooks/database/templates/default/ebs-db-backup.sh.erb

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Back up a MySQL database via EBS snapshot
+
+. /mnt/aws-config/config
+
+/opt/ec2_mysql/bin/ec2_mysql -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -p '<%= @mysql_root_passwd %>' -k 5 master
+echo "done"

data/lib/cookbooks/database/templates/default/ebs-db-restore.sh.erb

@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# Restore a MySQL database from EBS
+
+mkdir -p /mnt/restore
+
+. /mnt/aws-config/config
+
+/opt/ec2_mysql/bin/ec2_mysql -a $AWS_ACCESS_KEY_ID -s $AWS_SECRET_ACCESS_KEY -p '<%= @mysql_root_password %>' -v '<%= @ebs_vol_id %>' -m /mnt/restore -d <%= @mysql_device %> -r <%= @mysql_device %> -l debug -n slave
+echo "done"

data/lib/cookbooks/database/templates/default/s3cfg.erb

@@ -0,0 +1,27 @@
+[default]
+access_key = <%= @aws['aws_access_key_id'] %>
+acl_public = False
+bucket_location = US
+debug_syncmatch = False
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encrypt = False
+force = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = False
+host_base = s3.amazonaws.com
+host_bucket = %(bucket)s.s3.amazonaws.com
+human_readable_sizes = False
+preserve_attrs = True
+proxy_host = 
+proxy_port = 0
+recv_chunk = 4096
+secret_key = <%= @aws['aws_secret_access_key'] %>
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+use_https = True
+verbosity = WARNING

data/lib/cookbooks/mysql/.gitignore

@@ -0,0 +1,5 @@
+.bundle
+.cache
+.kitchen
+bin
+*.sw[op]

data/lib/cookbooks/mysql/CHANGELOG.md

@@ -0,0 +1,86 @@
+## v1.3.0:
+
+**Important note for this release**
+
+This version no longer installs Ruby bindings in the client recipe by
+default. Use the ruby recipe if you'd like the RubyGem. If you'd like
+packages from your distribution, use them in your application's
+specific cookbook/recipe, or modify the client packages attribute.
+This resolves the following tickets:
+
+* COOK-932
+* COOK-1009
+* COOK-1384
+
+Additionally, this cookbook now has tests (COOK-1439) for use under
+test-kitchen.
+
+The following issues are also addressed in this release.
+
+* [COOK-1443] - MySQL (>= 5.1.24) does not support `innodb_flush_method`
+  = fdatasync
+* [COOK-1175] - Add Mac OS X support
+* [COOK-1289] - handle additional tunable attributes
+* [COOK-1305] - add auto-increment-increment and auto-increment-offset
+  attributes
+* [COOK-1397] - make the port an attribute
+* [COOK-1439] - Add MySQL cookbook tests for test-kitchen support
+* [COOK-1236] - Move package names into attributes to allow percona to
+  free-ride
+* [COOK-934] - remove deprecated mysql/libraries/database.rb, use the
+  database cookbook instead.
+* [COOK-1475] - fix restart on config change
+
+## v1.2.6:
+
+* [COOK-1113] - Use an attribute to determine if upstart is used
+* [COOK-1121] - Add support for Windows
+* [COOK-1140] - Fix conf.d on Debian
+* [COOK-1151] - Fix server_ec2 handling /var/lib/mysql bind mount
+* [COOK-1321] - Document setting password attributes for solo
+
+## v1.2.4
+
+* [COOK-992] - fix FATAL nameerror
+* [COOK-827] - `mysql:server_ec2` recipe can't mount `data_dir`
+* [COOK-945] - FreeBSD support
+
+## v1.2.2
+
+* [COOK-826] mysql::server recipe doesn't quote password string
+* [COOK-834] Add 'scientific' and 'amazon' platforms to mysql cookbook
+
+## v1.2.1
+
+* [COOK-644] Mysql client cookbook 'package missing' error message is confusing
+* [COOK-645] RHEL6/CentOS6 - mysql cookbook contains 'skip-federated' directive which is unsupported on MySQL 5.1
+
+## v1.2.0
+
+* [COOK-684] remove mysql_database LWRP
+
+## v1.0.8:
+
+* [COOK-633] ensure "cloud" attribute is available
+
+## v1.0.7:
+
+* [COOK-614] expose all mysql tunable settings in config
+* [COOK-617] bind to private IP if available
+
+## v1.0.6:
+
+* [COOK-605] install mysql-client package on ubuntu/debian
+
+## v1.0.5:
+
+* [COOK-465] allow optional remote root connections to mysql
+* [COOK-455] improve platform version handling
+* externalize conf_dir attribute for easier cross platform support
+* change datadir attribute to data_dir for consistency
+
+## v1.0.4:
+
+* fix regressions on debian platform
+* [COOK-578] wrap root password in quotes
+* [COOK-562] expose all tunables in my.cnf

data/lib/cookbooks/mysql/CONTRIBUTING

@@ -0,0 +1,29 @@
+If you would like to contribute, please open a ticket in JIRA:
+
+* http://tickets.opscode.com
+
+Create the ticket in the COOK project and use the cookbook name as the
+component.
+
+For all code contributions, we ask that contributors sign a
+contributor license agreement (CLA). Instructions may be found here:
+
+* http://wiki.opscode.com/display/chef/How+to+Contribute
+
+When contributing changes to individual cookbooks, please do not
+modify the version number in the metadata.rb. Also please do not
+update the CHANGELOG.md for a new version. Not all changes to a
+cookbook may be merged and released in the same versions. Opscode will
+handle the version updates during the release process. You are welcome
+to correct typos or otherwise make updates to documentation in the
+README.
+
+If a contribution adds new platforms or platform versions, indicate
+such in the body of the commit message(s), and update the relevant
+COOK ticket. When writing commit messages, it is helpful for others if
+you indicate the COOK ticket. For example:
+
+    git commit -m '[COOK-1041] Updated pool resource to correctly delete.'
+
+In the ticket itself, it is also helpful if you include log output of
+a successful Chef run, but this is not absolutely required.

data/lib/cookbooks/mysql/Gemfile

@@ -0,0 +1,8 @@
+source :rubygems
+
+gem 'cucumber', '~> 1.1.8'
+gem 'minitest', '~> 3.0.0'
+
+group :kitchen  do
+  gem 'test-kitchen'
+end