aws_helper 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +71 -71
- data/aws_helper.gemspec +34 -34
- data/bin/aws_helper +4 -4
- data/lib/awshelper/cli.rb +408 -408
- data/lib/awshelper/ec2.rb +142 -142
- data/lib/awshelper/elb.rb +12 -12
- data/lib/awshelper/version.rb +3 -3
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f2806fb004098139d6ffc4f668c09a82de8e28f0
|
|
4
|
+
data.tar.gz: 4e2fc911c4223486a84605abdbb00dd0f887336a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: eb3b1ac8870f7f1f7707ebd582e038f46a5dbd846b5f791ff6861d635fa159951a6dd872445cd37f0862c8a2b7094ad2b885764316d0296100119647d355c019
|
|
7
|
+
data.tar.gz: 4c732aaa201185aba89796cca80632eb364c3f8e6362e3cea87c18edb2d533bc5c0fb428fd213c6d64fa193884da7c93b825f0b8864aff571470c594bb822e68
|
data/README.md
CHANGED
|
@@ -1,71 +1,71 @@
|
|
|
1
|
-
# aws_helper
|
|
2
|
-
|
|
3
|
-
Aws Helper for an instance
|
|
4
|
-
|
|
5
|
-
Allows functions on EBS volumes, snapshots, IP addresses and more
|
|
6
|
-
* initially snapshots are supported
|
|
7
|
-
|
|
8
|
-
## Installation
|
|
9
|
-
|
|
10
|
-
Add this line to your application's Gemfile:
|
|
11
|
-
|
|
12
|
-
gem 'aws_helper'
|
|
13
|
-
|
|
14
|
-
And then execute:
|
|
15
|
-
|
|
16
|
-
$ bundle
|
|
17
|
-
|
|
18
|
-
Or install it yourself as:
|
|
19
|
-
|
|
20
|
-
$ gem install aws_helper
|
|
21
|
-
|
|
22
|
-
## Minimal Usage
|
|
23
|
-
|
|
24
|
-
Assuming server start with an IAM role that have read access to AWS can create and delete snapshots:
|
|
25
|
-
|
|
26
|
-
Snapshot EBS root device at /dev/sda1
|
|
27
|
-
|
|
28
|
-
aws_helper snap /dev/sda1 --description zzzzzzzzz
|
|
29
|
-
|
|
30
|
-
Prune so only keep 7 snapshots:
|
|
31
|
-
|
|
32
|
-
aws_helper snap_prune /dev/sda1 --snapshots_to_keep=7
|
|
33
|
-
|
|
34
|
-
Email me a list of the latest 20 snapshots:
|
|
35
|
-
|
|
36
|
-
aws_helper snap_email me@company.com ebs.backups@company.com mysmtpemailserver.com
|
|
37
|
-
|
|
38
|
-
Cleanup ebs disks - Delete old server root disks:
|
|
39
|
-
|
|
40
|
-
aws_helper ebs_cleanup
|
|
41
|
-
|
|
42
|
-
Disks that are 8GB in size, not attached to a server, not tagged in any way and from a snapshot will be deleted.
|
|
43
|
-
|
|
44
|
-
## Complex Usage
|
|
45
|
-
|
|
46
|
-
If your server does not have a role then you need to code the AWS keys which is not best practice:
|
|
47
|
-
|
|
48
|
-
Snapshot EBS attached to device /dev/sdf volume vol-123456 access AWS through an http proxy:
|
|
49
|
-
|
|
50
|
-
export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
51
|
-
export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
52
|
-
export HTTP_PROXY=http://myproxy:port
|
|
53
|
-
aws_helper snap /dev/sdf vol-123456 --description zzzzzzzzz
|
|
54
|
-
|
|
55
|
-
Prune so only keep 20 snapshots:
|
|
56
|
-
|
|
57
|
-
export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
58
|
-
export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
59
|
-
export HTTP_PROXY=http://myproxy:port
|
|
60
|
-
aws_helper snap_prune /dev/sdf vol-123456 --snapshots_to_keep=20
|
|
61
|
-
|
|
62
|
-
Email me a list of the latest 30 snapshots with a subject title on email:
|
|
63
|
-
|
|
64
|
-
export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
65
|
-
export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
66
|
-
export HTTP_PROXY=http://myproxy:port
|
|
67
|
-
aws_helper snap_email me@company.com ebs.backups@company.com mysmtpemailserver.com 'My EBS Backups' --rows=30
|
|
68
|
-
|
|
69
|
-
Other functions to follow
|
|
70
|
-
|
|
71
|
-
|
|
1
|
+
# aws_helper
|
|
2
|
+
|
|
3
|
+
Aws Helper for an instance
|
|
4
|
+
|
|
5
|
+
Allows functions on EBS volumes, snapshots, IP addresses and more
|
|
6
|
+
* initially snapshots are supported
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
Add this line to your application's Gemfile:
|
|
11
|
+
|
|
12
|
+
gem 'aws_helper'
|
|
13
|
+
|
|
14
|
+
And then execute:
|
|
15
|
+
|
|
16
|
+
$ bundle
|
|
17
|
+
|
|
18
|
+
Or install it yourself as:
|
|
19
|
+
|
|
20
|
+
$ gem install aws_helper
|
|
21
|
+
|
|
22
|
+
## Minimal Usage
|
|
23
|
+
|
|
24
|
+
Assuming server start with an IAM role that have read access to AWS can create and delete snapshots:
|
|
25
|
+
|
|
26
|
+
Snapshot EBS root device at /dev/sda1
|
|
27
|
+
|
|
28
|
+
aws_helper snap /dev/sda1 --description zzzzzzzzz
|
|
29
|
+
|
|
30
|
+
Prune so only keep 7 snapshots:
|
|
31
|
+
|
|
32
|
+
aws_helper snap_prune /dev/sda1 --snapshots_to_keep=7
|
|
33
|
+
|
|
34
|
+
Email me a list of the latest 20 snapshots:
|
|
35
|
+
|
|
36
|
+
aws_helper snap_email me@company.com ebs.backups@company.com mysmtpemailserver.com
|
|
37
|
+
|
|
38
|
+
Cleanup ebs disks - Delete old server root disks:
|
|
39
|
+
|
|
40
|
+
aws_helper ebs_cleanup
|
|
41
|
+
|
|
42
|
+
Disks that are 8GB in size, not attached to a server, not tagged in any way and from a snapshot will be deleted.
|
|
43
|
+
|
|
44
|
+
## Complex Usage
|
|
45
|
+
|
|
46
|
+
If your server does not have a role then you need to code the AWS keys which is not best practice:
|
|
47
|
+
|
|
48
|
+
Snapshot EBS attached to device /dev/sdf volume vol-123456 access AWS through an http proxy:
|
|
49
|
+
|
|
50
|
+
export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
51
|
+
export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
52
|
+
export HTTP_PROXY=http://myproxy:port
|
|
53
|
+
aws_helper snap /dev/sdf vol-123456 --description zzzzzzzzz
|
|
54
|
+
|
|
55
|
+
Prune so only keep 20 snapshots:
|
|
56
|
+
|
|
57
|
+
export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
58
|
+
export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
59
|
+
export HTTP_PROXY=http://myproxy:port
|
|
60
|
+
aws_helper snap_prune /dev/sdf vol-123456 --snapshots_to_keep=20
|
|
61
|
+
|
|
62
|
+
Email me a list of the latest 30 snapshots with a subject title on email:
|
|
63
|
+
|
|
64
|
+
export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
65
|
+
export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
66
|
+
export HTTP_PROXY=http://myproxy:port
|
|
67
|
+
aws_helper snap_email me@company.com ebs.backups@company.com mysmtpemailserver.com 'My EBS Backups' --rows=30
|
|
68
|
+
|
|
69
|
+
Other functions to follow
|
|
70
|
+
|
|
71
|
+
|
data/aws_helper.gemspec
CHANGED
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
lib = File.expand_path('../lib', __FILE__)
|
|
4
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
|
-
require 'awshelper/version'
|
|
6
|
-
|
|
7
|
-
Gem::Specification.new do |s|
|
|
8
|
-
s.name = 'aws_helper'
|
|
9
|
-
s.version = Awshelper::VERSION
|
|
10
|
-
s.authors = ['Neill Turner']
|
|
11
|
-
s.email = ['neillwturner@gmail.com']
|
|
12
|
-
s.homepage = 'https://github.com/neillturner/aws_helper'
|
|
13
|
-
s.summary = 'Aws Helper for an instance'
|
|
14
|
-
candidates = Dir.glob('{lib}/**/*') + ['README.md', 'aws_helper.gemspec']
|
|
15
|
-
candidates = candidates + Dir.glob("bin/*")
|
|
16
|
-
s.files = candidates.sort
|
|
17
|
-
s.platform = Gem::Platform::RUBY
|
|
18
|
-
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
19
|
-
s.require_paths = ['lib']
|
|
20
|
-
s.add_dependency('right_aws')
|
|
21
|
-
s.add_dependency('thor')
|
|
22
|
-
s.rubyforge_project = '[none]'
|
|
23
|
-
s.description = <<-EOF
|
|
24
|
-
== DESCRIPTION:
|
|
25
|
-
|
|
26
|
-
Aws Helper for an instance
|
|
27
|
-
|
|
28
|
-
== FEATURES:
|
|
29
|
-
|
|
30
|
-
Allows functions on EBS volumes, snapshots, IP addresses and more
|
|
31
|
-
|
|
32
|
-
EOF
|
|
33
|
-
|
|
34
|
-
end
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('../lib', __FILE__)
|
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
|
+
require 'awshelper/version'
|
|
6
|
+
|
|
7
|
+
Gem::Specification.new do |s|
|
|
8
|
+
s.name = 'aws_helper'
|
|
9
|
+
s.version = Awshelper::VERSION
|
|
10
|
+
s.authors = ['Neill Turner']
|
|
11
|
+
s.email = ['neillwturner@gmail.com']
|
|
12
|
+
s.homepage = 'https://github.com/neillturner/aws_helper'
|
|
13
|
+
s.summary = 'Aws Helper for an instance'
|
|
14
|
+
candidates = Dir.glob('{lib}/**/*') + ['README.md', 'aws_helper.gemspec']
|
|
15
|
+
candidates = candidates + Dir.glob("bin/*")
|
|
16
|
+
s.files = candidates.sort
|
|
17
|
+
s.platform = Gem::Platform::RUBY
|
|
18
|
+
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
19
|
+
s.require_paths = ['lib']
|
|
20
|
+
s.add_dependency('right_aws')
|
|
21
|
+
s.add_dependency('thor')
|
|
22
|
+
s.rubyforge_project = '[none]'
|
|
23
|
+
s.description = <<-EOF
|
|
24
|
+
== DESCRIPTION:
|
|
25
|
+
|
|
26
|
+
Aws Helper for an instance
|
|
27
|
+
|
|
28
|
+
== FEATURES:
|
|
29
|
+
|
|
30
|
+
Allows functions on EBS volumes, snapshots, IP addresses and more
|
|
31
|
+
|
|
32
|
+
EOF
|
|
33
|
+
|
|
34
|
+
end
|
data/bin/aws_helper
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
|
|
3
|
-
require "awshelper/cli"
|
|
4
|
-
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "awshelper/cli"
|
|
4
|
+
|
|
5
5
|
Awshelper::CLI.start(ARGV)
|
data/lib/awshelper/cli.rb
CHANGED
|
@@ -1,408 +1,408 @@
|
|
|
1
|
-
require 'thor'
|
|
2
|
-
require 'awshelper'
|
|
3
|
-
require 'awshelper/ec2'
|
|
4
|
-
require 'syslog'
|
|
5
|
-
require 'net/smtp'
|
|
6
|
-
require 'json'
|
|
7
|
-
|
|
8
|
-
module Awshelper
|
|
9
|
-
class CLI < Thor
|
|
10
|
-
include Thor::Actions
|
|
11
|
-
|
|
12
|
-
include Awshelper::Ec2
|
|
13
|
-
|
|
14
|
-
#def ebs_create(volume_id, snapshot_id, most_recent_snapshot)
|
|
15
|
-
# #TO DO
|
|
16
|
-
# raise "Cannot create a volume with a specific id (EC2 chooses volume ids)" if volume_id
|
|
17
|
-
# if snapshot_id =~ /vol/
|
|
18
|
-
# new_resource.snapshot_id(find_snapshot_id(new_resource.snapshot_id, new_resource.most_recent_snapshot))
|
|
19
|
-
# end
|
|
20
|
-
#
|
|
21
|
-
# #nvid = volume_id_in_node_data
|
|
22
|
-
# #if nvid
|
|
23
|
-
# # # volume id is registered in the node data, so check that the volume in fact exists in EC2
|
|
24
|
-
# # vol = volume_by_id(nvid)
|
|
25
|
-
# # exists = vol && vol[:aws_status] != "deleting"
|
|
26
|
-
# # # TODO: determine whether this should be an error or just cause a new volume to be created. Currently erring on the side of failing loudly
|
|
27
|
-
# # raise "Volume with id #{nvid} is registered with the node but does not exist in EC2. To clear this error, remove the ['aws']['ebs_volume']['#{new_resource.name}']['volume_id'] entry from this node's data." unless exists
|
|
28
|
-
# #else
|
|
29
|
-
# # Determine if there is a volume that meets the resource's specifications and is attached to the current
|
|
30
|
-
# # instance in case a previous [:create, :attach] run created and attached a volume but for some reason was
|
|
31
|
-
# # not registered in the node data (e.g. an exception is thrown after the attach_volume request was accepted
|
|
32
|
-
# # by EC2, causing the node data to not be stored on the server)
|
|
33
|
-
# if new_resource.device && (attached_volume = currently_attached_volume(instance_id, new_resource.device))
|
|
34
|
-
# Chef::Log.debug("There is already a volume attached at device #{new_resource.device}")
|
|
35
|
-
# compatible = volume_compatible_with_resource_definition?(attached_volume)
|
|
36
|
-
# raise "Volume #{attached_volume[:aws_id]} attached at #{attached_volume[:aws_device]} but does not conform to this resource's specifications" unless compatible
|
|
37
|
-
# Chef::Log.debug("The volume matches the resource's definition, so the volume is assumed to be already created")
|
|
38
|
-
# converge_by("update the node data with volume id: #{attached_volume[:aws_id]}") do
|
|
39
|
-
# node.set['aws']['ebs_volume'][new_resource.name]['volume_id'] = attached_volume[:aws_id]
|
|
40
|
-
# node.save unless Chef::Config[:solo]
|
|
41
|
-
# end
|
|
42
|
-
# else
|
|
43
|
-
# # If not, create volume and register its id in the node data
|
|
44
|
-
# converge_by("create a volume with id=#{new_resource.snapshot_id} size=#{new_resource.size} availability_zone=#{new_resource.availability_zone} and update the node data with created volume's id") do
|
|
45
|
-
# nvid = create_volume(new_resource.snapshot_id,
|
|
46
|
-
# new_resource.size,
|
|
47
|
-
# new_resource.availability_zone,
|
|
48
|
-
# new_resource.timeout,
|
|
49
|
-
# new_resource.volume_type,
|
|
50
|
-
# new_resource.piops)
|
|
51
|
-
# node.set['aws']['ebs_volume'][new_resource.name]['volume_id'] = nvid
|
|
52
|
-
# node.save unless Chef::Config[:solo]
|
|
53
|
-
# end
|
|
54
|
-
# end
|
|
55
|
-
# #end
|
|
56
|
-
#end
|
|
57
|
-
|
|
58
|
-
#def ebs_attach(device, volume_id, timeout)
|
|
59
|
-
# # determine_volume returns a Hash, not a Mash, and the keys are
|
|
60
|
-
# # symbols, not strings.
|
|
61
|
-
# vol = determine_volume(device, volume_id)
|
|
62
|
-
# if vol[:aws_status] == "in-use"
|
|
63
|
-
# if vol[:aws_instance_id] != instance_id
|
|
64
|
-
# raise "Volume with id #{vol[:aws_id]} exists but is attached to instance #{vol[:aws_instance_id]}"
|
|
65
|
-
# else
|
|
66
|
-
# Chef::Log.debug("Volume is already attached")
|
|
67
|
-
# end
|
|
68
|
-
# else
|
|
69
|
-
# # attach the volume
|
|
70
|
-
# attach_volume(vol[:aws_id], instance_id, device, timeout)
|
|
71
|
-
# end
|
|
72
|
-
#end
|
|
73
|
-
|
|
74
|
-
#def ebs_detach(device, volume_id, timeout)
|
|
75
|
-
# vol = determine_volume(device, volume_id)
|
|
76
|
-
# detach_volume(vol[:aws_id], timeout)
|
|
77
|
-
#end
|
|
78
|
-
|
|
79
|
-
desc "snap DEVICE [VOLUME_ID]", "Take a snapshot of a EBS Disk."
|
|
80
|
-
option :description
|
|
81
|
-
|
|
82
|
-
long_desc <<-LONGDESC
|
|
83
|
-
'snap DEVICE [VOLUME_ID] --description xxxxxx'
|
|
84
|
-
\x5 Take a snapshot of a EBS Disk by specifying device and/or volume_id.
|
|
85
|
-
\x5 All commands rely on environment variables or the server having an IAM role
|
|
86
|
-
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxx'
|
|
87
|
-
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyy'
|
|
88
|
-
\x5 For example
|
|
89
|
-
\x5 aws_helper snap /dev/sdf
|
|
90
|
-
\x5 will snap shot the EBS disk attach to device /dev/xvdj
|
|
91
|
-
LONGDESC
|
|
92
|
-
|
|
93
|
-
def snap(device, volume_id=nil)
|
|
94
|
-
vol = determine_volume(device, volume_id)
|
|
95
|
-
snap_description = options[:description] if options[:description]
|
|
96
|
-
snap_description = "Created by aws_helper(#{instance_id}/#{local_ipv4}) for #{ami_id} from #{vol[:aws_id]}" if !options[:description]
|
|
97
|
-
snapshot = ec2.create_snapshot(vol[:aws_id],snap_description)
|
|
98
|
-
log("Created snapshot of #{vol[:aws_id]} as #{snapshot[:aws_id]}")
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
desc "snap_prune DEVICE [VOLUME_ID]", "Prune the number of snapshots."
|
|
102
|
-
option :snapshots_to_keep, :type => :numeric, :required => true
|
|
103
|
-
|
|
104
|
-
long_desc <<-LONGDESC
|
|
105
|
-
'snap_prune DEVICE [VOLUME_ID] --snapshots_to_keep=<numeric>'
|
|
106
|
-
\x5 Prune the number of snapshots of a EBS Disk by specifying device and/or volume_id and the no to keep.
|
|
107
|
-
\x5 All commands rely on environment variables or the server having an IAM role
|
|
108
|
-
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
109
|
-
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
110
|
-
\x5 For example
|
|
111
|
-
\x5 aws_helper snap_prune /dev/sdf --snapshots_to_keep=7
|
|
112
|
-
\x5 will keep the last 7 snapshots of the EBS disk attach to device /dev/xvdj
|
|
113
|
-
LONGDESC
|
|
114
|
-
|
|
115
|
-
def snap_prune(device, volume_id=nil)
|
|
116
|
-
snapshots_to_keep = options[:snapshots_to_keep]
|
|
117
|
-
vol = determine_volume(device, volume_id)
|
|
118
|
-
old_snapshots = Array.new
|
|
119
|
-
log("Checking for old snapshots")
|
|
120
|
-
ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
|
|
121
|
-
if snapshot[:aws_volume_id] == vol[:aws_id]
|
|
122
|
-
log("Found old snapshot #{snapshot[:aws_id]} (#{snapshot[:aws_volume_id]}) #{snapshot[:aws_started_at]}")
|
|
123
|
-
old_snapshots << snapshot
|
|
124
|
-
end
|
|
125
|
-
end
|
|
126
|
-
if old_snapshots.length > snapshots_to_keep
|
|
127
|
-
old_snapshots[snapshots_to_keep, old_snapshots.length].each do |die|
|
|
128
|
-
log("Deleting old snapshot #{die[:aws_id]}")
|
|
129
|
-
ec2.delete_snapshot(die[:aws_id])
|
|
130
|
-
end
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
desc "snap_email TO FROM EMAIL_SERVER", "Email Snapshot List."
|
|
135
|
-
option :rows, :type => :numeric, :required => false
|
|
136
|
-
option :owner, :type => :numeric, :required => false
|
|
137
|
-
|
|
138
|
-
long_desc <<-LONGDESC
|
|
139
|
-
'snap_email TO FROM EMAIL_SERVER ['EBS Backups'] --rows=<numeric> --owner=<numeric>'
|
|
140
|
-
\x5 Emails the last 20 snapshots from specific email address via the email_server.
|
|
141
|
-
\x5 All commands rely on environment variables or the server having an IAM role
|
|
142
|
-
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
143
|
-
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
144
|
-
\x5 For example
|
|
145
|
-
\x5 aws_helper snap_email me@mycompany.com ebs.backups@mycompany.com emailserver.com 'My EBS Backups' --rows=20 -owner=999887777
|
|
146
|
-
\x5 will email the list of the latest 20 snapshots to email address me@mycompany.com via email server emailserver.com
|
|
147
|
-
\x5 that belong to aws owner 999887777
|
|
148
|
-
LONGDESC
|
|
149
|
-
|
|
150
|
-
def snap_email(to, from, email_server, subject='EBS Backups')
|
|
151
|
-
rows = 20
|
|
152
|
-
rows = options[:rows] if options[:rows]
|
|
153
|
-
owner = {}
|
|
154
|
-
owner = {:
|
|
155
|
-
message = ""
|
|
156
|
-
log("Report on snapshots")
|
|
157
|
-
# ({ Name="start-time", Values="today in YYYY-MM-DD"})
|
|
158
|
-
i = rows
|
|
159
|
-
ec2.describe_snapshots(owner).sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
|
|
160
|
-
if i >0
|
|
161
|
-
message = message+"#{snapshot[:aws_id]} #{snapshot[:aws_volume_id]} #{snapshot[:aws_started_at]} #{snapshot[:aws_description]} #{snapshot[:aws_status]}\n"
|
|
162
|
-
i = i-1
|
|
163
|
-
end
|
|
164
|
-
end
|
|
165
|
-
opts = {}
|
|
166
|
-
opts[:server] = email_server
|
|
167
|
-
opts[:from] = from
|
|
168
|
-
opts[:from_alias] = 'EBS Backups'
|
|
169
|
-
opts[:subject] = subject
|
|
170
|
-
opts[:body] = message
|
|
171
|
-
send_email(to,opts)
|
|
172
|
-
end
|
|
173
|
-
|
|
174
|
-
desc "ebs_cleanup", "Cleanup ebs disks - Delete old server root disks."
|
|
175
|
-
|
|
176
|
-
long_desc <<-LONGDESC
|
|
177
|
-
'ebs_cleanup'
|
|
178
|
-
\x5 Cleanup ebs disks - Delete old server root disks.
|
|
179
|
-
\x5 Disks that are 8GB in size, not attached to a server, not tagged in any way and from a snapshot.
|
|
180
|
-
\x5 All commands rely on environment variables or the server having an IAM role.
|
|
181
|
-
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
182
|
-
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
183
|
-
\x5 For example
|
|
184
|
-
\x5 ebs_cleanup
|
|
185
|
-
LONGDESC
|
|
186
|
-
|
|
187
|
-
def ebs_cleanup()
|
|
188
|
-
ec2.describe_volumes(:filters => { 'status' => 'available', 'size' => '8' }).each do |r|
|
|
189
|
-
if r[:aws_size] == 8 and r[:aws_status] == 'available' and r[:tags] == {} and r[:snapshot_id] != nil and r[:snapshot_id][0,5] == 'snap-' then
|
|
190
|
-
log("Deleting unused volume #{r[:aws_id]} from snapshot #{r[:snapshot_id]}")
|
|
191
|
-
ec2.delete_volume(r[:aws_id])
|
|
192
|
-
end
|
|
193
|
-
end
|
|
194
|
-
end
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
private
|
|
198
|
-
|
|
199
|
-
def log(message,type="info")
|
|
200
|
-
# $0 is the current script name
|
|
201
|
-
puts message
|
|
202
|
-
Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "info"
|
|
203
|
-
Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "err"
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
# Pulls the volume id from the volume_id attribute or the node data and verifies that the volume actually exists
|
|
207
|
-
def determine_volume(device, volume_id)
|
|
208
|
-
vol = currently_attached_volume(instance_id, device)
|
|
209
|
-
vol_id = volume_id || ( vol ? vol[:aws_id] : nil )
|
|
210
|
-
log("volume_id attribute not set and no volume is attached at the device #{device}",'err') unless vol_id
|
|
211
|
-
raise "volume_id attribute not set and no volume is attached at the device #{device}" unless vol_id
|
|
212
|
-
|
|
213
|
-
# check that volume exists
|
|
214
|
-
vol = volume_by_id(vol_id)
|
|
215
|
-
log("No volume with id #{vol_id} exists",'err') unless vol
|
|
216
|
-
raise "No volume with id #{vol_id} exists" unless vol
|
|
217
|
-
|
|
218
|
-
vol
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
def get_all_instances(filter={})
|
|
223
|
-
data = []
|
|
224
|
-
response = ec2.describe_instances(filter)
|
|
225
|
-
if response.status == 200
|
|
226
|
-
data_s = response.body['reservationSet']
|
|
227
|
-
data_s.each do |rs|
|
|
228
|
-
gs=rs['groupSet']
|
|
229
|
-
rs['instancesSet'].each do |r|
|
|
230
|
-
#r[:aws_instance_id] = r['instanceId']
|
|
231
|
-
#r[:public_ip] = r['ipAddress']
|
|
232
|
-
#r[:aws_state] = r['instanceState']['name']
|
|
233
|
-
#r['groupSet']=rs['groupSet']
|
|
234
|
-
data.push(r)
|
|
235
|
-
end
|
|
236
|
-
end
|
|
237
|
-
end
|
|
238
|
-
data
|
|
239
|
-
end
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
# Retrieves information for a volume
|
|
243
|
-
def volume_by_id(volume_id)
|
|
244
|
-
ec2.describe_volumes.find{|v| v[:aws_id] == volume_id}
|
|
245
|
-
end
|
|
246
|
-
|
|
247
|
-
# Returns the volume that's attached to the instance at the given device or nil if none matches
|
|
248
|
-
def currently_attached_volume(instance_id, device)
|
|
249
|
-
ec2.describe_volumes.find{|v| v[:aws_instance_id] == instance_id && v[:aws_device] == device}
|
|
250
|
-
end
|
|
251
|
-
|
|
252
|
-
# Returns true if the given volume meets the resource's attributes
|
|
253
|
-
#def volume_compatible_with_resource_definition?(volume)
|
|
254
|
-
# if new_resource.snapshot_id =~ /vol/
|
|
255
|
-
# new_resource.snapshot_id(find_snapshot_id(new_resource.snapshot_id, new_resource.most_recent_snapshot))
|
|
256
|
-
# end
|
|
257
|
-
# (new_resource.size.nil? || new_resource.size == volume[:aws_size]) &&
|
|
258
|
-
# (new_resource.availability_zone.nil? || new_resource.availability_zone == volume[:zone]) &&
|
|
259
|
-
# (new_resource.snapshot_id.nil? || new_resource.snapshot_id == volume[:snapshot_id])
|
|
260
|
-
#end
|
|
261
|
-
|
|
262
|
-
# TODO: support tags in deswcription
|
|
263
|
-
#def tag_value(instance,tag_key)
|
|
264
|
-
# options = ec2.describe_tags({:filters => {:resource_id => instance }} )
|
|
265
|
-
# end
|
|
266
|
-
|
|
267
|
-
# Creates a volume according to specifications and blocks until done (or times out)
|
|
268
|
-
def create_volume(snapshot_id, size, availability_zone, timeout, volume_type, piops)
|
|
269
|
-
availability_zone ||= instance_availability_zone
|
|
270
|
-
|
|
271
|
-
# Sanity checks so we don't shoot ourselves.
|
|
272
|
-
raise "Invalid volume type: #{volume_type}" unless ['standard', 'io1', 'gp2'].include?(volume_type)
|
|
273
|
-
|
|
274
|
-
# PIOPs requested. Must specify an iops param and probably won't be "low".
|
|
275
|
-
if volume_type == 'io1'
|
|
276
|
-
raise 'IOPS value not specified.' unless piops >= 100
|
|
277
|
-
end
|
|
278
|
-
|
|
279
|
-
# Shouldn't see non-zero piops param without appropriate type.
|
|
280
|
-
if piops > 0
|
|
281
|
-
raise 'IOPS param without piops volume type.' unless volume_type == 'io1'
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
create_volume_opts = { :volume_type => volume_type }
|
|
285
|
-
# TODO: this may have to be casted to a string. rightaws vs aws doc discrepancy.
|
|
286
|
-
create_volume_opts[:iops] = piops if volume_type == 'io1'
|
|
287
|
-
|
|
288
|
-
nv = ec2.create_volume(snapshot_id, size, availability_zone, create_volume_opts)
|
|
289
|
-
Chef::Log.debug("Created new volume #{nv[:aws_id]}#{snapshot_id ? " based on #{snapshot_id}" : ""}")
|
|
290
|
-
|
|
291
|
-
# block until created
|
|
292
|
-
begin
|
|
293
|
-
Timeout::timeout(timeout) do
|
|
294
|
-
while true
|
|
295
|
-
vol = volume_by_id(nv[:aws_id])
|
|
296
|
-
if vol && vol[:aws_status] != "deleting"
|
|
297
|
-
if ["in-use", "available"].include?(vol[:aws_status])
|
|
298
|
-
Chef::Log.info("Volume #{nv[:aws_id]} is available")
|
|
299
|
-
break
|
|
300
|
-
else
|
|
301
|
-
Chef::Log.debug("Volume is #{vol[:aws_status]}")
|
|
302
|
-
end
|
|
303
|
-
sleep 3
|
|
304
|
-
else
|
|
305
|
-
raise "Volume #{nv[:aws_id]} no longer exists"
|
|
306
|
-
end
|
|
307
|
-
end
|
|
308
|
-
end
|
|
309
|
-
rescue Timeout::Error
|
|
310
|
-
raise "Timed out waiting for volume creation after #{timeout} seconds"
|
|
311
|
-
end
|
|
312
|
-
|
|
313
|
-
nv[:aws_id]
|
|
314
|
-
end
|
|
315
|
-
|
|
316
|
-
# Attaches the volume and blocks until done (or times out)
|
|
317
|
-
def attach_volume(volume_id, instance_id, device, timeout)
|
|
318
|
-
Chef::Log.debug("Attaching #{volume_id} as #{device}")
|
|
319
|
-
ec2.attach_volume(volume_id, instance_id, device)
|
|
320
|
-
|
|
321
|
-
# block until attached
|
|
322
|
-
begin
|
|
323
|
-
Timeout::timeout(timeout) do
|
|
324
|
-
while true
|
|
325
|
-
vol = volume_by_id(volume_id)
|
|
326
|
-
if vol && vol[:aws_status] != "deleting"
|
|
327
|
-
if vol[:aws_attachment_status] == "attached"
|
|
328
|
-
if vol[:aws_instance_id] == instance_id
|
|
329
|
-
Chef::Log.info("Volume #{volume_id} is attached to #{instance_id}")
|
|
330
|
-
break
|
|
331
|
-
else
|
|
332
|
-
raise "Volume is attached to instance #{vol[:aws_instance_id]} instead of #{instance_id}"
|
|
333
|
-
end
|
|
334
|
-
else
|
|
335
|
-
Chef::Log.debug("Volume is #{vol[:aws_status]}")
|
|
336
|
-
end
|
|
337
|
-
sleep 3
|
|
338
|
-
else
|
|
339
|
-
raise "Volume #{volume_id} no longer exists"
|
|
340
|
-
end
|
|
341
|
-
end
|
|
342
|
-
end
|
|
343
|
-
rescue Timeout::Error
|
|
344
|
-
raise "Timed out waiting for volume attachment after #{timeout} seconds"
|
|
345
|
-
end
|
|
346
|
-
end
|
|
347
|
-
|
|
348
|
-
# Detaches the volume and blocks until done (or times out)
|
|
349
|
-
def detach_volume(volume_id, timeout)
|
|
350
|
-
vol = volume_by_id(volume_id)
|
|
351
|
-
if vol[:aws_instance_id] != instance_id
|
|
352
|
-
Chef::Log.debug("EBS Volume #{volume_id} is not attached to this instance (attached to #{vol[:aws_instance_id]}). Skipping...")
|
|
353
|
-
return
|
|
354
|
-
end
|
|
355
|
-
Chef::Log.debug("Detaching #{volume_id}")
|
|
356
|
-
orig_instance_id = vol[:aws_instance_id]
|
|
357
|
-
ec2.detach_volume(volume_id)
|
|
358
|
-
|
|
359
|
-
# block until detached
|
|
360
|
-
begin
|
|
361
|
-
Timeout::timeout(timeout) do
|
|
362
|
-
while true
|
|
363
|
-
vol = volume_by_id(volume_id)
|
|
364
|
-
if vol && vol[:aws_status] != "deleting"
|
|
365
|
-
if vol[:aws_instance_id] != orig_instance_id
|
|
366
|
-
Chef::Log.info("Volume detached from #{orig_instance_id}")
|
|
367
|
-
break
|
|
368
|
-
else
|
|
369
|
-
Chef::Log.debug("Volume: #{vol.inspect}")
|
|
370
|
-
end
|
|
371
|
-
else
|
|
372
|
-
Chef::Log.debug("Volume #{volume_id} no longer exists")
|
|
373
|
-
break
|
|
374
|
-
end
|
|
375
|
-
sleep 3
|
|
376
|
-
end
|
|
377
|
-
end
|
|
378
|
-
rescue Timeout::Error
|
|
379
|
-
raise "Timed out waiting for volume detachment after #{timeout} seconds"
|
|
380
|
-
end
|
|
381
|
-
end
|
|
382
|
-
|
|
383
|
-
def send_email(to,opts={})
|
|
384
|
-
opts[:server] ||= 'localhost'
|
|
385
|
-
opts[:from] ||= 'email@example.com'
|
|
386
|
-
opts[:from_alias] ||= 'Example Emailer'
|
|
387
|
-
opts[:subject] ||= "You need to see this"
|
|
388
|
-
opts[:body] ||= "Important stuff!"
|
|
389
|
-
|
|
390
|
-
msg = <<END_OF_MESSAGE
|
|
391
|
-
From: #{opts[:from_alias]} <#{opts[:from]}>
|
|
392
|
-
To: <#{to}>
|
|
393
|
-
Subject: #{opts[:subject]}
|
|
394
|
-
|
|
395
|
-
#{opts[:body]}
|
|
396
|
-
END_OF_MESSAGE
|
|
397
|
-
puts "Sending to #{to} from #{opts[:from]} email server #{opts[:server]}"
|
|
398
|
-
Net::SMTP.start(opts[:server]) do |smtp|
|
|
399
|
-
smtp.send_message msg, opts[:from], to
|
|
400
|
-
end
|
|
401
|
-
end
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
end
|
|
405
|
-
|
|
406
|
-
end
|
|
407
|
-
|
|
408
|
-
|
|
1
|
+
require 'thor'
|
|
2
|
+
require 'awshelper'
|
|
3
|
+
require 'awshelper/ec2'
|
|
4
|
+
require 'syslog'
|
|
5
|
+
require 'net/smtp'
|
|
6
|
+
require 'json'
|
|
7
|
+
|
|
8
|
+
module Awshelper
|
|
9
|
+
class CLI < Thor
|
|
10
|
+
include Thor::Actions
|
|
11
|
+
|
|
12
|
+
include Awshelper::Ec2
|
|
13
|
+
|
|
14
|
+
#def ebs_create(volume_id, snapshot_id, most_recent_snapshot)
|
|
15
|
+
# #TO DO
|
|
16
|
+
# raise "Cannot create a volume with a specific id (EC2 chooses volume ids)" if volume_id
|
|
17
|
+
# if snapshot_id =~ /vol/
|
|
18
|
+
# new_resource.snapshot_id(find_snapshot_id(new_resource.snapshot_id, new_resource.most_recent_snapshot))
|
|
19
|
+
# end
|
|
20
|
+
#
|
|
21
|
+
# #nvid = volume_id_in_node_data
|
|
22
|
+
# #if nvid
|
|
23
|
+
# # # volume id is registered in the node data, so check that the volume in fact exists in EC2
|
|
24
|
+
# # vol = volume_by_id(nvid)
|
|
25
|
+
# # exists = vol && vol[:aws_status] != "deleting"
|
|
26
|
+
# # # TODO: determine whether this should be an error or just cause a new volume to be created. Currently erring on the side of failing loudly
|
|
27
|
+
# # raise "Volume with id #{nvid} is registered with the node but does not exist in EC2. To clear this error, remove the ['aws']['ebs_volume']['#{new_resource.name}']['volume_id'] entry from this node's data." unless exists
|
|
28
|
+
# #else
|
|
29
|
+
# # Determine if there is a volume that meets the resource's specifications and is attached to the current
|
|
30
|
+
# # instance in case a previous [:create, :attach] run created and attached a volume but for some reason was
|
|
31
|
+
# # not registered in the node data (e.g. an exception is thrown after the attach_volume request was accepted
|
|
32
|
+
# # by EC2, causing the node data to not be stored on the server)
|
|
33
|
+
# if new_resource.device && (attached_volume = currently_attached_volume(instance_id, new_resource.device))
|
|
34
|
+
# Chef::Log.debug("There is already a volume attached at device #{new_resource.device}")
|
|
35
|
+
# compatible = volume_compatible_with_resource_definition?(attached_volume)
|
|
36
|
+
# raise "Volume #{attached_volume[:aws_id]} attached at #{attached_volume[:aws_device]} but does not conform to this resource's specifications" unless compatible
|
|
37
|
+
# Chef::Log.debug("The volume matches the resource's definition, so the volume is assumed to be already created")
|
|
38
|
+
# converge_by("update the node data with volume id: #{attached_volume[:aws_id]}") do
|
|
39
|
+
# node.set['aws']['ebs_volume'][new_resource.name]['volume_id'] = attached_volume[:aws_id]
|
|
40
|
+
# node.save unless Chef::Config[:solo]
|
|
41
|
+
# end
|
|
42
|
+
# else
|
|
43
|
+
# # If not, create volume and register its id in the node data
|
|
44
|
+
# converge_by("create a volume with id=#{new_resource.snapshot_id} size=#{new_resource.size} availability_zone=#{new_resource.availability_zone} and update the node data with created volume's id") do
|
|
45
|
+
# nvid = create_volume(new_resource.snapshot_id,
|
|
46
|
+
# new_resource.size,
|
|
47
|
+
# new_resource.availability_zone,
|
|
48
|
+
# new_resource.timeout,
|
|
49
|
+
# new_resource.volume_type,
|
|
50
|
+
# new_resource.piops)
|
|
51
|
+
# node.set['aws']['ebs_volume'][new_resource.name]['volume_id'] = nvid
|
|
52
|
+
# node.save unless Chef::Config[:solo]
|
|
53
|
+
# end
|
|
54
|
+
# end
|
|
55
|
+
# #end
|
|
56
|
+
#end
|
|
57
|
+
|
|
58
|
+
#def ebs_attach(device, volume_id, timeout)
|
|
59
|
+
# # determine_volume returns a Hash, not a Mash, and the keys are
|
|
60
|
+
# # symbols, not strings.
|
|
61
|
+
# vol = determine_volume(device, volume_id)
|
|
62
|
+
# if vol[:aws_status] == "in-use"
|
|
63
|
+
# if vol[:aws_instance_id] != instance_id
|
|
64
|
+
# raise "Volume with id #{vol[:aws_id]} exists but is attached to instance #{vol[:aws_instance_id]}"
|
|
65
|
+
# else
|
|
66
|
+
# Chef::Log.debug("Volume is already attached")
|
|
67
|
+
# end
|
|
68
|
+
# else
|
|
69
|
+
# # attach the volume
|
|
70
|
+
# attach_volume(vol[:aws_id], instance_id, device, timeout)
|
|
71
|
+
# end
|
|
72
|
+
#end
|
|
73
|
+
|
|
74
|
+
#def ebs_detach(device, volume_id, timeout)
|
|
75
|
+
# vol = determine_volume(device, volume_id)
|
|
76
|
+
# detach_volume(vol[:aws_id], timeout)
|
|
77
|
+
#end
|
|
78
|
+
|
|
79
|
+
desc "snap DEVICE [VOLUME_ID]", "Take a snapshot of a EBS Disk."
|
|
80
|
+
option :description
|
|
81
|
+
|
|
82
|
+
long_desc <<-LONGDESC
|
|
83
|
+
'snap DEVICE [VOLUME_ID] --description xxxxxx'
|
|
84
|
+
\x5 Take a snapshot of a EBS Disk by specifying device and/or volume_id.
|
|
85
|
+
\x5 All commands rely on environment variables or the server having an IAM role
|
|
86
|
+
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxx'
|
|
87
|
+
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyy'
|
|
88
|
+
\x5 For example
|
|
89
|
+
\x5 aws_helper snap /dev/sdf
|
|
90
|
+
\x5 will snap shot the EBS disk attach to device /dev/xvdj
|
|
91
|
+
LONGDESC
|
|
92
|
+
|
|
93
|
+
def snap(device, volume_id=nil)
|
|
94
|
+
vol = determine_volume(device, volume_id)
|
|
95
|
+
snap_description = options[:description] if options[:description]
|
|
96
|
+
snap_description = "Created by aws_helper(#{instance_id}/#{local_ipv4}) for #{ami_id} from #{vol[:aws_id]}" if !options[:description]
|
|
97
|
+
snapshot = ec2.create_snapshot(vol[:aws_id],snap_description)
|
|
98
|
+
log("Created snapshot of #{vol[:aws_id]} as #{snapshot[:aws_id]}")
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
desc "snap_prune DEVICE [VOLUME_ID]", "Prune the number of snapshots."
|
|
102
|
+
option :snapshots_to_keep, :type => :numeric, :required => true
|
|
103
|
+
|
|
104
|
+
long_desc <<-LONGDESC
|
|
105
|
+
'snap_prune DEVICE [VOLUME_ID] --snapshots_to_keep=<numeric>'
|
|
106
|
+
\x5 Prune the number of snapshots of a EBS Disk by specifying device and/or volume_id and the no to keep.
|
|
107
|
+
\x5 All commands rely on environment variables or the server having an IAM role
|
|
108
|
+
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
109
|
+
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
110
|
+
\x5 For example
|
|
111
|
+
\x5 aws_helper snap_prune /dev/sdf --snapshots_to_keep=7
|
|
112
|
+
\x5 will keep the last 7 snapshots of the EBS disk attach to device /dev/xvdj
|
|
113
|
+
LONGDESC
|
|
114
|
+
|
|
115
|
+
def snap_prune(device, volume_id=nil)
|
|
116
|
+
snapshots_to_keep = options[:snapshots_to_keep]
|
|
117
|
+
vol = determine_volume(device, volume_id)
|
|
118
|
+
old_snapshots = Array.new
|
|
119
|
+
log("Checking for old snapshots")
|
|
120
|
+
ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
|
|
121
|
+
if snapshot[:aws_volume_id] == vol[:aws_id]
|
|
122
|
+
log("Found old snapshot #{snapshot[:aws_id]} (#{snapshot[:aws_volume_id]}) #{snapshot[:aws_started_at]}")
|
|
123
|
+
old_snapshots << snapshot
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
if old_snapshots.length > snapshots_to_keep
|
|
127
|
+
old_snapshots[snapshots_to_keep, old_snapshots.length].each do |die|
|
|
128
|
+
log("Deleting old snapshot #{die[:aws_id]}")
|
|
129
|
+
ec2.delete_snapshot(die[:aws_id])
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
desc "snap_email TO FROM EMAIL_SERVER", "Email Snapshot List."
|
|
135
|
+
option :rows, :type => :numeric, :required => false
|
|
136
|
+
option :owner, :type => :numeric, :required => false
|
|
137
|
+
|
|
138
|
+
long_desc <<-LONGDESC
|
|
139
|
+
'snap_email TO FROM EMAIL_SERVER ['EBS Backups'] --rows=<numeric> --owner=<numeric>'
|
|
140
|
+
\x5 Emails the last 20 snapshots from specific email address via the email_server.
|
|
141
|
+
\x5 All commands rely on environment variables or the server having an IAM role
|
|
142
|
+
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
143
|
+
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
144
|
+
\x5 For example
|
|
145
|
+
\x5 aws_helper snap_email me@mycompany.com ebs.backups@mycompany.com emailserver.com 'My EBS Backups' --rows=20 -owner=999887777
|
|
146
|
+
\x5 will email the list of the latest 20 snapshots to email address me@mycompany.com via email server emailserver.com
|
|
147
|
+
\x5 that belong to aws owner 999887777
|
|
148
|
+
LONGDESC
|
|
149
|
+
|
|
150
|
+
def snap_email(to, from, email_server, subject='EBS Backups')
|
|
151
|
+
rows = 20
|
|
152
|
+
rows = options[:rows] if options[:rows]
|
|
153
|
+
owner = {}
|
|
154
|
+
owner = {:aws_owner => options[:owner]} if options[:owner]
|
|
155
|
+
message = ""
|
|
156
|
+
log("Report on snapshots")
|
|
157
|
+
# ({ Name="start-time", Values="today in YYYY-MM-DD"})
|
|
158
|
+
i = rows
|
|
159
|
+
ec2.describe_snapshots(owner).sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
|
|
160
|
+
if i >0
|
|
161
|
+
message = message+"#{snapshot[:aws_id]} #{snapshot[:aws_volume_id]} #{snapshot[:aws_started_at]} #{snapshot[:aws_description]} #{snapshot[:aws_status]}\n"
|
|
162
|
+
i = i-1
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
opts = {}
|
|
166
|
+
opts[:server] = email_server
|
|
167
|
+
opts[:from] = from
|
|
168
|
+
opts[:from_alias] = 'EBS Backups'
|
|
169
|
+
opts[:subject] = subject
|
|
170
|
+
opts[:body] = message
|
|
171
|
+
send_email(to,opts)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
desc "ebs_cleanup", "Cleanup ebs disks - Delete old server root disks."
|
|
175
|
+
|
|
176
|
+
long_desc <<-LONGDESC
|
|
177
|
+
'ebs_cleanup'
|
|
178
|
+
\x5 Cleanup ebs disks - Delete old server root disks.
|
|
179
|
+
\x5 Disks that are 8GB in size, not attached to a server, not tagged in any way and from a snapshot.
|
|
180
|
+
\x5 All commands rely on environment variables or the server having an IAM role.
|
|
181
|
+
\x5 export AWS_ACCESS_KEY_ID ='xxxxxxxxxxxx'
|
|
182
|
+
\x5 export AWS_SECRET_ACCESS_KEY ='yyyyyyyy'
|
|
183
|
+
\x5 For example
|
|
184
|
+
\x5 ebs_cleanup
|
|
185
|
+
LONGDESC
|
|
186
|
+
|
|
187
|
+
def ebs_cleanup()
|
|
188
|
+
ec2.describe_volumes(:filters => { 'status' => 'available', 'size' => '8' }).each do |r|
|
|
189
|
+
if r[:aws_size] == 8 and r[:aws_status] == 'available' and r[:tags] == {} and r[:snapshot_id] != nil and r[:snapshot_id][0,5] == 'snap-' then
|
|
190
|
+
log("Deleting unused volume #{r[:aws_id]} from snapshot #{r[:snapshot_id]}")
|
|
191
|
+
ec2.delete_volume(r[:aws_id])
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
private
|
|
198
|
+
|
|
199
|
+
def log(message,type="info")
|
|
200
|
+
# $0 is the current script name
|
|
201
|
+
puts message
|
|
202
|
+
Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "info"
|
|
203
|
+
Syslog.open($0, Syslog::LOG_PID | Syslog::LOG_CONS) { |s| s.info message } if type == "err"
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Pulls the volume id from the volume_id attribute or the node data and verifies that the volume actually exists
|
|
207
|
+
def determine_volume(device, volume_id)
|
|
208
|
+
vol = currently_attached_volume(instance_id, device)
|
|
209
|
+
vol_id = volume_id || ( vol ? vol[:aws_id] : nil )
|
|
210
|
+
log("volume_id attribute not set and no volume is attached at the device #{device}",'err') unless vol_id
|
|
211
|
+
raise "volume_id attribute not set and no volume is attached at the device #{device}" unless vol_id
|
|
212
|
+
|
|
213
|
+
# check that volume exists
|
|
214
|
+
vol = volume_by_id(vol_id)
|
|
215
|
+
log("No volume with id #{vol_id} exists",'err') unless vol
|
|
216
|
+
raise "No volume with id #{vol_id} exists" unless vol
|
|
217
|
+
|
|
218
|
+
vol
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def get_all_instances(filter={})
|
|
223
|
+
data = []
|
|
224
|
+
response = ec2.describe_instances(filter)
|
|
225
|
+
if response.status == 200
|
|
226
|
+
data_s = response.body['reservationSet']
|
|
227
|
+
data_s.each do |rs|
|
|
228
|
+
gs=rs['groupSet']
|
|
229
|
+
rs['instancesSet'].each do |r|
|
|
230
|
+
#r[:aws_instance_id] = r['instanceId']
|
|
231
|
+
#r[:public_ip] = r['ipAddress']
|
|
232
|
+
#r[:aws_state] = r['instanceState']['name']
|
|
233
|
+
#r['groupSet']=rs['groupSet']
|
|
234
|
+
data.push(r)
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
data
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
# Retrieves information for a volume
|
|
243
|
+
def volume_by_id(volume_id)
|
|
244
|
+
ec2.describe_volumes.find{|v| v[:aws_id] == volume_id}
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# Returns the volume that's attached to the instance at the given device or nil if none matches
|
|
248
|
+
def currently_attached_volume(instance_id, device)
|
|
249
|
+
ec2.describe_volumes.find{|v| v[:aws_instance_id] == instance_id && v[:aws_device] == device}
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Returns true if the given volume meets the resource's attributes
|
|
253
|
+
#def volume_compatible_with_resource_definition?(volume)
|
|
254
|
+
# if new_resource.snapshot_id =~ /vol/
|
|
255
|
+
# new_resource.snapshot_id(find_snapshot_id(new_resource.snapshot_id, new_resource.most_recent_snapshot))
|
|
256
|
+
# end
|
|
257
|
+
# (new_resource.size.nil? || new_resource.size == volume[:aws_size]) &&
|
|
258
|
+
# (new_resource.availability_zone.nil? || new_resource.availability_zone == volume[:zone]) &&
|
|
259
|
+
# (new_resource.snapshot_id.nil? || new_resource.snapshot_id == volume[:snapshot_id])
|
|
260
|
+
#end
|
|
261
|
+
|
|
262
|
+
# TODO: support tags in deswcription
|
|
263
|
+
#def tag_value(instance,tag_key)
|
|
264
|
+
# options = ec2.describe_tags({:filters => {:resource_id => instance }} )
|
|
265
|
+
# end
|
|
266
|
+
|
|
267
|
+
# Creates a volume according to specifications and blocks until done (or times out)
|
|
268
|
+
def create_volume(snapshot_id, size, availability_zone, timeout, volume_type, piops)
|
|
269
|
+
availability_zone ||= instance_availability_zone
|
|
270
|
+
|
|
271
|
+
# Sanity checks so we don't shoot ourselves.
|
|
272
|
+
raise "Invalid volume type: #{volume_type}" unless ['standard', 'io1', 'gp2'].include?(volume_type)
|
|
273
|
+
|
|
274
|
+
# PIOPs requested. Must specify an iops param and probably won't be "low".
|
|
275
|
+
if volume_type == 'io1'
|
|
276
|
+
raise 'IOPS value not specified.' unless piops >= 100
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Shouldn't see non-zero piops param without appropriate type.
|
|
280
|
+
if piops > 0
|
|
281
|
+
raise 'IOPS param without piops volume type.' unless volume_type == 'io1'
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
create_volume_opts = { :volume_type => volume_type }
|
|
285
|
+
# TODO: this may have to be casted to a string. rightaws vs aws doc discrepancy.
|
|
286
|
+
create_volume_opts[:iops] = piops if volume_type == 'io1'
|
|
287
|
+
|
|
288
|
+
nv = ec2.create_volume(snapshot_id, size, availability_zone, create_volume_opts)
|
|
289
|
+
Chef::Log.debug("Created new volume #{nv[:aws_id]}#{snapshot_id ? " based on #{snapshot_id}" : ""}")
|
|
290
|
+
|
|
291
|
+
# block until created
|
|
292
|
+
begin
|
|
293
|
+
Timeout::timeout(timeout) do
|
|
294
|
+
while true
|
|
295
|
+
vol = volume_by_id(nv[:aws_id])
|
|
296
|
+
if vol && vol[:aws_status] != "deleting"
|
|
297
|
+
if ["in-use", "available"].include?(vol[:aws_status])
|
|
298
|
+
Chef::Log.info("Volume #{nv[:aws_id]} is available")
|
|
299
|
+
break
|
|
300
|
+
else
|
|
301
|
+
Chef::Log.debug("Volume is #{vol[:aws_status]}")
|
|
302
|
+
end
|
|
303
|
+
sleep 3
|
|
304
|
+
else
|
|
305
|
+
raise "Volume #{nv[:aws_id]} no longer exists"
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
rescue Timeout::Error
|
|
310
|
+
raise "Timed out waiting for volume creation after #{timeout} seconds"
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
nv[:aws_id]
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# Attaches the volume and blocks until done (or times out)
|
|
317
|
+
def attach_volume(volume_id, instance_id, device, timeout)
|
|
318
|
+
Chef::Log.debug("Attaching #{volume_id} as #{device}")
|
|
319
|
+
ec2.attach_volume(volume_id, instance_id, device)
|
|
320
|
+
|
|
321
|
+
# block until attached
|
|
322
|
+
begin
|
|
323
|
+
Timeout::timeout(timeout) do
|
|
324
|
+
while true
|
|
325
|
+
vol = volume_by_id(volume_id)
|
|
326
|
+
if vol && vol[:aws_status] != "deleting"
|
|
327
|
+
if vol[:aws_attachment_status] == "attached"
|
|
328
|
+
if vol[:aws_instance_id] == instance_id
|
|
329
|
+
Chef::Log.info("Volume #{volume_id} is attached to #{instance_id}")
|
|
330
|
+
break
|
|
331
|
+
else
|
|
332
|
+
raise "Volume is attached to instance #{vol[:aws_instance_id]} instead of #{instance_id}"
|
|
333
|
+
end
|
|
334
|
+
else
|
|
335
|
+
Chef::Log.debug("Volume is #{vol[:aws_status]}")
|
|
336
|
+
end
|
|
337
|
+
sleep 3
|
|
338
|
+
else
|
|
339
|
+
raise "Volume #{volume_id} no longer exists"
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
end
|
|
343
|
+
rescue Timeout::Error
|
|
344
|
+
raise "Timed out waiting for volume attachment after #{timeout} seconds"
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# Detaches the volume and blocks until done (or times out)
|
|
349
|
+
def detach_volume(volume_id, timeout)
|
|
350
|
+
vol = volume_by_id(volume_id)
|
|
351
|
+
if vol[:aws_instance_id] != instance_id
|
|
352
|
+
Chef::Log.debug("EBS Volume #{volume_id} is not attached to this instance (attached to #{vol[:aws_instance_id]}). Skipping...")
|
|
353
|
+
return
|
|
354
|
+
end
|
|
355
|
+
Chef::Log.debug("Detaching #{volume_id}")
|
|
356
|
+
orig_instance_id = vol[:aws_instance_id]
|
|
357
|
+
ec2.detach_volume(volume_id)
|
|
358
|
+
|
|
359
|
+
# block until detached
|
|
360
|
+
begin
|
|
361
|
+
Timeout::timeout(timeout) do
|
|
362
|
+
while true
|
|
363
|
+
vol = volume_by_id(volume_id)
|
|
364
|
+
if vol && vol[:aws_status] != "deleting"
|
|
365
|
+
if vol[:aws_instance_id] != orig_instance_id
|
|
366
|
+
Chef::Log.info("Volume detached from #{orig_instance_id}")
|
|
367
|
+
break
|
|
368
|
+
else
|
|
369
|
+
Chef::Log.debug("Volume: #{vol.inspect}")
|
|
370
|
+
end
|
|
371
|
+
else
|
|
372
|
+
Chef::Log.debug("Volume #{volume_id} no longer exists")
|
|
373
|
+
break
|
|
374
|
+
end
|
|
375
|
+
sleep 3
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
rescue Timeout::Error
|
|
379
|
+
raise "Timed out waiting for volume detachment after #{timeout} seconds"
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
def send_email(to,opts={})
|
|
384
|
+
opts[:server] ||= 'localhost'
|
|
385
|
+
opts[:from] ||= 'email@example.com'
|
|
386
|
+
opts[:from_alias] ||= 'Example Emailer'
|
|
387
|
+
opts[:subject] ||= "You need to see this"
|
|
388
|
+
opts[:body] ||= "Important stuff!"
|
|
389
|
+
|
|
390
|
+
msg = <<END_OF_MESSAGE
|
|
391
|
+
From: #{opts[:from_alias]} <#{opts[:from]}>
|
|
392
|
+
To: <#{to}>
|
|
393
|
+
Subject: #{opts[:subject]}
|
|
394
|
+
|
|
395
|
+
#{opts[:body]}
|
|
396
|
+
END_OF_MESSAGE
|
|
397
|
+
puts "Sending to #{to} from #{opts[:from]} email server #{opts[:server]}"
|
|
398
|
+
Net::SMTP.start(opts[:server]) do |smtp|
|
|
399
|
+
smtp.send_message msg, opts[:from], to
|
|
400
|
+
end
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
|