cloudblocks 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/plugins/logrotate.rb +61 -0
- data/lib/plugins/rotater.rb +9 -7
- data/lib/plugins/s3backup.rb +101 -0
- metadata +25 -12
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'quartz_plugin')
|
|
2
|
+
require 'fileutils'
|
|
3
|
+
|
|
4
|
+
class Logrotate < QuartzPlugin
|
|
5
|
+
|
|
6
|
+
def info
|
|
7
|
+
{ :uid => "8f4286bfd946c8b08b234833673b8860", :name => "Log Rotate", :version => "0.0.0" }
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def run(message)
|
|
11
|
+
pl = payload(message)
|
|
12
|
+
|
|
13
|
+
@source_pattern = pl['source pattern']
|
|
14
|
+
@dest_folder = pl['destination']
|
|
15
|
+
@keep = pl['keep'].empty? ? 0 : pl['keep'].to_i
|
|
16
|
+
@post_run_step = pl['post rotate']
|
|
17
|
+
|
|
18
|
+
ext = Time.now.utc.strftime('%Y%m%d%H%M%S')
|
|
19
|
+
|
|
20
|
+
FileUtils.mkdir_p(@dest_folder)
|
|
21
|
+
|
|
22
|
+
Dir.glob(@source_pattern).each do |f|
|
|
23
|
+
dest_file = File.join(@dest_folder, File.basename(f))
|
|
24
|
+
next if File.directory?(f)
|
|
25
|
+
fname = "#{dest_file}.gz"
|
|
26
|
+
dump_cmd = "cat #{f} | gzip > '#{fname}.#{ext}'"
|
|
27
|
+
@log.debug "Running #{dump_cmd}"
|
|
28
|
+
copy_result = run_shell(dump_cmd)
|
|
29
|
+
|
|
30
|
+
return run_result(false, copy_result[:message]) unless copy_result[:ok]
|
|
31
|
+
|
|
32
|
+
@log.debug "Removing source log files"
|
|
33
|
+
remove_result = run_shell("rm #{f}")
|
|
34
|
+
|
|
35
|
+
return run_result(false, remove_result[:message]) unless remove_result[:ok]
|
|
36
|
+
|
|
37
|
+
# find all the files from this one's rotations
|
|
38
|
+
rotated_pattern = "#{fname}.*"
|
|
39
|
+
@log.debug "Looking for rotated files #{rotated_pattern}"
|
|
40
|
+
all_rotated = Dir.glob(rotated_pattern)
|
|
41
|
+
if all_rotated.count > @keep
|
|
42
|
+
remove_count = all_rotated.count - @keep
|
|
43
|
+
to_remote = all_rotated.sort! { |a,b| File.mtime(a) <=> File.mtime(b) }[0...remove_count]
|
|
44
|
+
@log.debug "Removing extra #{remove_count} files"
|
|
45
|
+
to_remote.each do |tr|
|
|
46
|
+
@log.debug "Removing #{tr}"
|
|
47
|
+
remove_shell = run_shell("rm #{tr}")
|
|
48
|
+
return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
if !@post_run_step.nil? && !@post_run_step.empty?
|
|
54
|
+
@log.debug "Running post rotate step #{post_run_step}"
|
|
55
|
+
post_shell = run_shell(@post_run_step)
|
|
56
|
+
return run_result(false, post_shell[:message]) unless post_shell[:ok]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
return run_result(true, "Log files rotated succesfully")
|
|
60
|
+
end
|
|
61
|
+
end
|
data/lib/plugins/rotater.rb
CHANGED
|
@@ -31,6 +31,8 @@ class Rotater < QuartzPlugin
|
|
|
31
31
|
move_shell = run_shell("mv #{f} #{dest_path}")
|
|
32
32
|
return run_result(false, move_shell[:message]) unless move_shell[:ok]
|
|
33
33
|
|
|
34
|
+
return run_result(true, "Files moved successfully with no rotation") if keep == 0
|
|
35
|
+
|
|
34
36
|
# find all the files from this one's rotations
|
|
35
37
|
rotated_pattern = "#{File.join(archive, fname)}.*"
|
|
36
38
|
@log.debug "Looking for rotated files #{rotated_pattern}"
|
|
@@ -38,16 +40,16 @@ class Rotater < QuartzPlugin
|
|
|
38
40
|
if all_rotated.count > keep
|
|
39
41
|
remove_count = all_rotated.count - keep
|
|
40
42
|
to_remote = all_rotated.sort! { |a,b| File.mtime(a) <=> File.mtime(b) }[0...remove_count]
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
43
|
+
@log.debug "Removing extra files"
|
|
44
|
+
to_remote.each do |tr|
|
|
45
|
+
@log.debug "Removing #{tr}"
|
|
46
|
+
remove_shell = run_shell("rm #{tr}")
|
|
47
|
+
return run_result(false, remove_shell[:message]) unless remove_shell[:ok]
|
|
48
|
+
end
|
|
47
49
|
end
|
|
48
50
|
end
|
|
49
51
|
|
|
50
|
-
if post_rotate && rotated
|
|
52
|
+
if !post_rotate.empty? && rotated
|
|
51
53
|
@log.debug "Running post rotate step #{post_rotate}"
|
|
52
54
|
post_shell = run_shell(post_rotate)
|
|
53
55
|
return run_result(false, post_shell[:message]) unless post_shell[:ok]
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
require File.join(File.dirname(__FILE__), 'quartz_plugin')
|
|
2
|
+
require 'fileutils'
|
|
3
|
+
require 'fog'
|
|
4
|
+
|
|
5
|
+
class S3backup < QuartzPlugin
|
|
6
|
+
|
|
7
|
+
def info
|
|
8
|
+
{ :uid => "d3533989f9d542f393566511e8eb2090", :name => "S3 Backup", :version => "0.0.0" }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run(message)
|
|
12
|
+
pl = payload(message)
|
|
13
|
+
|
|
14
|
+
@log.debug "Pruned payload #{pl}"
|
|
15
|
+
|
|
16
|
+
@access_key_id = pl['access key']
|
|
17
|
+
@secret_access_key = pl['secret key']
|
|
18
|
+
@bucket = pl['bucket']
|
|
19
|
+
@remote_path = pl['remote path']
|
|
20
|
+
@region = pl['region']
|
|
21
|
+
@local_pattern = pl['local pattern']
|
|
22
|
+
@keep = pl['keep'].empty? ? 0 : pl['keep'].to_i
|
|
23
|
+
|
|
24
|
+
@testing = pl['testing']
|
|
25
|
+
|
|
26
|
+
return transfer
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def connection
|
|
32
|
+
Fog.mock! unless @testing.nil? || @testing == false
|
|
33
|
+
@connection ||= Fog::Storage.new(
|
|
34
|
+
:provider => 'AWS',
|
|
35
|
+
:aws_access_key_id => @access_key_id,
|
|
36
|
+
:aws_secret_access_key => @secret_access_key,
|
|
37
|
+
:region => @region
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def remote_path_for(filename)
|
|
42
|
+
filename.sub(/^\//, '')
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def transfer
|
|
46
|
+
begin
|
|
47
|
+
remote_path = remote_path_for(@remote_path)
|
|
48
|
+
@log.debug "Remote path is #{remote_path}"
|
|
49
|
+
|
|
50
|
+
@log.debug "Syncronizing local and remote clocks"
|
|
51
|
+
connection.sync_clock
|
|
52
|
+
|
|
53
|
+
count = 0
|
|
54
|
+
# get local files
|
|
55
|
+
Dir.glob(@local_pattern).each do |f|
|
|
56
|
+
remote_file = File.join(@remote_path, File.basename(f))
|
|
57
|
+
next if File.directory?(f)
|
|
58
|
+
@log.debug "Copying #{f} to #{remote_file}"
|
|
59
|
+
count += 1
|
|
60
|
+
File.open(f, 'r') do |file|
|
|
61
|
+
connection.put_object(@bucket, File.join(remote_path, File.basename(f)), file)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
return run_result(true, "Files copied to S3 bucket successfully with no rotation") if @keep == 0
|
|
66
|
+
|
|
67
|
+
directory = connection.directories.get(@bucket)
|
|
68
|
+
all_rotated = directory.files.reject { |m| File.dirname(m.key) != @remote_path }
|
|
69
|
+
@log.debug "Found #{all_rotated.count} in the remote bucket"
|
|
70
|
+
if all_rotated.count > @keep
|
|
71
|
+
remove_count = all_rotated.count - @keep
|
|
72
|
+
@log.debug "Removing #{remove_count} and keeping the most recent #{@keep}"
|
|
73
|
+
to_remove = all_rotated.sort { |a,b| a.last_modified <=> b.last_modified }.map{|m| m.key }[0...remove_count]
|
|
74
|
+
@log.debug "Removing extra files"
|
|
75
|
+
to_remove.each do |tr|
|
|
76
|
+
@log.debug "Removing #{tr}"
|
|
77
|
+
connection.delete_object(@bucket, tr)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
rescue Excon::Errors::SocketError => exc
|
|
81
|
+
@log.error exc.message
|
|
82
|
+
return run_result(false, exc.message)
|
|
83
|
+
rescue Excon::Errors::Error => exc
|
|
84
|
+
@log.error exc.message
|
|
85
|
+
result = exc.response.body
|
|
86
|
+
message = result.match(/\<Message\>(.*)\<\/Message\>/)
|
|
87
|
+
if !message.nil?
|
|
88
|
+
message = message[1]
|
|
89
|
+
return run_result(false, message)
|
|
90
|
+
elsif exc.response.status == 404
|
|
91
|
+
return run_result(false, "Remote S3 serivce or bucket not found (404)")
|
|
92
|
+
elsif exc.response.status != 0
|
|
93
|
+
return run_result(false, "Remote S3 serivce returned error #{exc.response.status} without any more details")
|
|
94
|
+
else
|
|
95
|
+
return run_result(false, exc.message)
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
run_result(true, "Successfully copied #{count} files to S3")
|
|
100
|
+
end
|
|
101
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cloudblocks
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.8
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -9,11 +9,11 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2012-03-
|
|
12
|
+
date: 2012-03-20 00:00:00.000000000Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: httparty
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70274217198280 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ! '>='
|
|
@@ -21,10 +21,10 @@ dependencies:
|
|
|
21
21
|
version: 0.8.1
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70274217198280
|
|
25
25
|
- !ruby/object:Gem::Dependency
|
|
26
26
|
name: json
|
|
27
|
-
requirement: &
|
|
27
|
+
requirement: &70274217197400 !ruby/object:Gem::Requirement
|
|
28
28
|
none: false
|
|
29
29
|
requirements:
|
|
30
30
|
- - ! '>='
|
|
@@ -32,10 +32,10 @@ dependencies:
|
|
|
32
32
|
version: 1.6.3
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
|
-
version_requirements: *
|
|
35
|
+
version_requirements: *70274217197400
|
|
36
36
|
- !ruby/object:Gem::Dependency
|
|
37
37
|
name: eventmachine
|
|
38
|
-
requirement: &
|
|
38
|
+
requirement: &70274217196640 !ruby/object:Gem::Requirement
|
|
39
39
|
none: false
|
|
40
40
|
requirements:
|
|
41
41
|
- - ! '>='
|
|
@@ -43,10 +43,10 @@ dependencies:
|
|
|
43
43
|
version: 0.12.10
|
|
44
44
|
type: :runtime
|
|
45
45
|
prerelease: false
|
|
46
|
-
version_requirements: *
|
|
46
|
+
version_requirements: *70274217196640
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
48
|
name: faye
|
|
49
|
-
requirement: &
|
|
49
|
+
requirement: &70274217195940 !ruby/object:Gem::Requirement
|
|
50
50
|
none: false
|
|
51
51
|
requirements:
|
|
52
52
|
- - ! '>='
|
|
@@ -54,10 +54,10 @@ dependencies:
|
|
|
54
54
|
version: 0.8.0
|
|
55
55
|
type: :runtime
|
|
56
56
|
prerelease: false
|
|
57
|
-
version_requirements: *
|
|
57
|
+
version_requirements: *70274217195940
|
|
58
58
|
- !ruby/object:Gem::Dependency
|
|
59
59
|
name: open4
|
|
60
|
-
requirement: &
|
|
60
|
+
requirement: &70274217195080 !ruby/object:Gem::Requirement
|
|
61
61
|
none: false
|
|
62
62
|
requirements:
|
|
63
63
|
- - ! '>='
|
|
@@ -65,7 +65,18 @@ dependencies:
|
|
|
65
65
|
version: 1.3.0
|
|
66
66
|
type: :runtime
|
|
67
67
|
prerelease: false
|
|
68
|
-
version_requirements: *
|
|
68
|
+
version_requirements: *70274217195080
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: fog
|
|
71
|
+
requirement: &70274217194380 !ruby/object:Gem::Requirement
|
|
72
|
+
none: false
|
|
73
|
+
requirements:
|
|
74
|
+
- - ! '>='
|
|
75
|
+
- !ruby/object:Gem::Version
|
|
76
|
+
version: 1.1.2
|
|
77
|
+
type: :runtime
|
|
78
|
+
prerelease: false
|
|
79
|
+
version_requirements: *70274217194380
|
|
69
80
|
description: See http://www.thecloudblocks.com for more info
|
|
70
81
|
email: khash@thecloudblocks.com
|
|
71
82
|
executables:
|
|
@@ -77,10 +88,12 @@ files:
|
|
|
77
88
|
- lib/config-chief.rb
|
|
78
89
|
- lib/cloud-quartz.rb
|
|
79
90
|
- lib/plugins/broken.rb
|
|
91
|
+
- lib/plugins/logrotate.rb
|
|
80
92
|
- lib/plugins/mysql.rb
|
|
81
93
|
- lib/plugins/quartz_plugin.rb
|
|
82
94
|
- lib/plugins/rake.rb
|
|
83
95
|
- lib/plugins/rotater.rb
|
|
96
|
+
- lib/plugins/s3backup.rb
|
|
84
97
|
- lib/plugins/shell.rb
|
|
85
98
|
- lib/plugins/tester.rb
|
|
86
99
|
- lib/plugins/webget.rb
|