awshucks 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGES +4 -0
- data/LICENSE +19 -0
- data/README +32 -0
- data/Rakefile +14 -0
- data/TODO +143 -0
- data/bin/awshucks +10 -0
- data/lib/awshucks.rb +29 -0
- data/lib/awshucks/cli.rb +76 -0
- data/lib/awshucks/command.rb +98 -0
- data/lib/awshucks/commands.rb +3 -0
- data/lib/awshucks/commands/backup.rb +59 -0
- data/lib/awshucks/commands/backups.rb +25 -0
- data/lib/awshucks/commands/help.rb +22 -0
- data/lib/awshucks/commands/list.rb +39 -0
- data/lib/awshucks/commands/new_config.rb +36 -0
- data/lib/awshucks/commands/reset_metadata_cache.rb +38 -0
- data/lib/awshucks/commands/restore.rb +54 -0
- data/lib/awshucks/config.rb +83 -0
- data/lib/awshucks/ext.rb +23 -0
- data/lib/awshucks/file_info.rb +23 -0
- data/lib/awshucks/file_store.rb +111 -0
- data/lib/awshucks/gemspec.rb +48 -0
- data/lib/awshucks/scanner.rb +58 -0
- data/lib/awshucks/specification.rb +128 -0
- data/lib/awshucks/version.rb +18 -0
- data/resources/awshucks.yml +21 -0
- data/spec/awshucks_spec.rb +7 -0
- data/spec/cli_spec.rb +130 -0
- data/spec/command_spec.rb +111 -0
- data/spec/commands/backup_spec.rb +164 -0
- data/spec/commands/backups_spec.rb +41 -0
- data/spec/commands/help_spec.rb +42 -0
- data/spec/commands/list_spec.rb +77 -0
- data/spec/commands/new_config_spec.rb +102 -0
- data/spec/commands/reset_metadata_cache_spec.rb +93 -0
- data/spec/commands/restore_spec.rb +219 -0
- data/spec/config_spec.rb +152 -0
- data/spec/ext_spec.rb +28 -0
- data/spec/file_info_spec.rb +45 -0
- data/spec/file_store_spec.rb +352 -0
- data/spec/scanner_spec.rb +106 -0
- data/spec/spec_helper.rb +36 -0
- data/spec/specification_spec.rb +41 -0
- metadata +121 -0
data/CHANGES
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Copyright (c) 2007 Nathan Witmer
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
4
|
+
this software and associated documentation files (the "Software"), to deal in
|
5
|
+
the Software without restriction, including without limitation the rights to
|
6
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
7
|
+
of the Software, and to permit persons to whom the Software is furnished to do
|
8
|
+
so, subject to the following conditions:
|
9
|
+
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
11
|
+
copies or substantial portions of the Software.
|
12
|
+
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
19
|
+
SOFTWARE.
|
data/README
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
== awshucks
|
2
|
+
|
3
|
+
* Homepage[http://awshucks.rubyforge.org/rabal/]
|
4
|
+
* {Rubyforge Project}[http://rubyforge.org/projects/awshucks]
|
5
|
+
* email nwitmer-rubyforge at otherward dot net
|
6
|
+
|
7
|
+
== DESCRIPTION
|
8
|
+
|
9
|
+
awshucks is a utility for backing up files to Amazon's Simple Storage Service (S3)
|
10
|
+
|
11
|
+
== INSTALL
|
12
|
+
|
13
|
+
gem install awshucks
|
14
|
+
|
15
|
+
== USAGE
|
16
|
+
|
17
|
+
awshucks help
|
18
|
+
|
19
|
+
== FEATURES
|
20
|
+
|
21
|
+
* back up specific directories
|
22
|
+
* caches file metadata for better performance
|
23
|
+
* stores all configured backups in a single bucket
|
24
|
+
* only uploads when files are new or changed
|
25
|
+
|
26
|
+
== CREDITS
|
27
|
+
|
28
|
+
Thanks to Jeremy Hinegardner and Tim Pease for the help getting started with my first gem!
|
29
|
+
|
30
|
+
== LICENSE
|
31
|
+
|
32
|
+
Released under the MIT license. See LICENSE.
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# make sure our project's ./lib directory is added to the ruby search path
|
2
|
+
$: << File.join(File.dirname(__FILE__),"lib")
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
require 'rake/gempackagetask'
|
6
|
+
require 'rake/clean'
|
7
|
+
require 'rake/rdoctask'
|
8
|
+
require 'rake/contrib/sshpublisher'
|
9
|
+
|
10
|
+
require 'awshucks'
|
11
|
+
|
12
|
+
load 'tasks/setup.rb'
|
13
|
+
|
14
|
+
|
data/TODO
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
Next:
|
2
|
+
|
3
|
+
* website and documentation
|
4
|
+
|
5
|
+
* look into marcel's usage of changes/changelog, and other automation
|
6
|
+
|
7
|
+
Awshucks
|
8
|
+
|
9
|
+
Awshucks is a utility for backing up your files to Amazon's http://aws.amazon.com/s3 Simple Storage Service (S3).
|
10
|
+
|
11
|
+
Getting started:
|
12
|
+
|
13
|
+
* Install awshucks via rubygems:
|
14
|
+
|
15
|
+
sudo gem install awshucks
|
16
|
+
|
17
|
+
* Generate a new config file
|
18
|
+
|
19
|
+
awshucks generate
|
20
|
+
|
21
|
+
This will generate a default config file called awshucks.yml into the current working directory.
|
22
|
+
|
23
|
+
* Edit the config file, specifically:
|
24
|
+
|
25
|
+
access_key_id: put your access key here
|
26
|
+
|
27
|
+
secret_access_key: put your secret access key here
|
28
|
+
|
29
|
+
bucket: bucket name where awshucks will store your backed-up files
|
30
|
+
|
31
|
+
* Add the locations you want to back up.
|
32
|
+
|
33
|
+
photos:
|
34
|
+
location: "~/pictures"
|
35
|
+
|
36
|
+
email:
|
37
|
+
location: "~/mail"
|
38
|
+
|
39
|
+
* Run a backup
|
40
|
+
|
41
|
+
awshucks backup
|
42
|
+
|
43
|
+
This will back up all of the files from the locations you specified.
|
44
|
+
|
45
|
+
Limitations:
|
46
|
+
|
47
|
+
* Awshucks will not ignore dotfiles or other files. It will not, however, follow symlinks.
|
48
|
+
* If you remove a file on the local filesystem, it will currently not be deleted from your bucket on S3.
|
49
|
+
* If you move a file on the local filesystem, awshucks will treat it as a new file and re-upload it.
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
|
54
|
+
Links in the sidebar:
|
55
|
+
|
56
|
+
download
|
57
|
+
project page
|
58
|
+
amazon s3
|
59
|
+
|
60
|
+
|
61
|
+
|
62
|
+
|
63
|
+
|
64
|
+
|
65
|
+
|
66
|
+
|
67
|
+
Todo:
|
68
|
+
|
69
|
+
* critical features before release:
|
70
|
+
* svn repo
|
71
|
+
|
72
|
+
* nice to have
|
73
|
+
* deletion of locally-removed files -- will need to flag scanned files somehow
|
74
|
+
* ignores
|
75
|
+
* better output/logging (cattr_accessor logger)
|
76
|
+
* archival of removed files -- same deal, but with move&rename with additional metadata
|
77
|
+
* list of backed up files
|
78
|
+
* list of archived files
|
79
|
+
* --dry-run -n flag
|
80
|
+
|
81
|
+
* ignores
|
82
|
+
* list ignores with backups
|
83
|
+
* allow list of ignore patterns in backup config
|
84
|
+
* set defaults in generated config file for ignore option, and automatically merge them with backup configs
|
85
|
+
|
86
|
+
* project
|
87
|
+
* update README, gemspec with name, version, etc. etc. including docs about how metadata is stored, how files are stored, etc. etc. (after a basic section about how things work at a high level)
|
88
|
+
* update README to include rspec as a dependency for running the test suite
|
89
|
+
* document: does not follow symlinks!
|
90
|
+
* sync svn repo to rubyforge: http://plans.inplanb.com/articles/2007/07/01/mirror-projects-to-a-read-only-svn-archive-with-svk
|
91
|
+
* write spec helper for input/output capturing?
|
92
|
+
* clean up the specs -- DRY up the mocks with helpers, etc.
|
93
|
+
|
94
|
+
* better error handling for file not found errors for file scanner (move exceptions to another file?)
|
95
|
+
* allow rename instead for files with matching md5s? smart algorithm for fixing data store stuff?
|
96
|
+
* refactor the redefinition of execute to make sure everything gets the same arguments? pppossibly, also with Command 'help' do ... ?
|
97
|
+
* add integration tests for each of the commands to make sure they get executed with the correct arguments (arity)
|
98
|
+
* DRY up the commands that take a backup config as an option
|
99
|
+
* DRY up the specs re: output handling -- can rspec descriptions have multiple before(:each) blocks?
|
100
|
+
* idea: constructor(s) take a block, and instance_eval it
|
101
|
+
|
102
|
+
Someday:
|
103
|
+
|
104
|
+
* AES encryption -- see keybox code, using Schneier's algorithm as a reference
|
105
|
+
* pgpme plugin / extra for encrypted backups (golgo)
|
106
|
+
|
107
|
+
Done:
|
108
|
+
|
109
|
+
* create rubyforge project
|
110
|
+
* figure out where to include various gems... probably best in the main file (yep, for now)
|
111
|
+
* rescue file-not-found error on -c flag
|
112
|
+
* find out how mtime (in about) gets set when uploading large files --> last-modified != file's mtime (IO stream)
|
113
|
+
* filesystem crawling
|
114
|
+
* reorganize command and specs
|
115
|
+
* remove alib dependency
|
116
|
+
* add backups command to list available backups and their configuration
|
117
|
+
* switch up IO handling in test cases and see if it's possible to replace dependence on CLI instance
|
118
|
+
* no more passing cli around at all!
|
119
|
+
* remove cli from args passed in to a command (piggyback help message onto config...)
|
120
|
+
* compute/store MD5sum of file on upload
|
121
|
+
* compare MD5 before uploading
|
122
|
+
* find out if ensure blocks will still get executed on SystemExit (they will)
|
123
|
+
* store metadata automatically as a separate file for better performance
|
124
|
+
* write code to actually use metadata instead of the file objects (most of the time)
|
125
|
+
* metadata: include bits of logic for handling touched-but-not-actually-changed files (update metadata cache)
|
126
|
+
* metadata: write "reset_metadata_cache" command, which takes an optional backup name as well -- deletes the metadata cache file
|
127
|
+
* write restore command
|
128
|
+
* default target, or specify a path
|
129
|
+
* compute/compare MD5sum of file on restore
|
130
|
+
* write list command
|
131
|
+
* generator for default config
|
132
|
+
|
133
|
+
Not Gonna Do:
|
134
|
+
|
135
|
+
* write sync_metadata command to use stored files' metadata as authoritative -- never mind, this happens implicitly.
|
136
|
+
|
137
|
+
Questions:
|
138
|
+
|
139
|
+
* can bucket names have spaces in them? --> no.
|
140
|
+
* how to handle deleted files? set metadata status? rename them with special naming and store metadata? (archive?) --> archive FTW!
|
141
|
+
* store metadata in special file about last-backed-up date and top-level directories? --> perhaps! it'd save bandwidth!
|
142
|
+
* warn if the same path is specified in two configs? pppossibly! --> ehhh not now, maybe later
|
143
|
+
* complain if a backup has a space in it? --> nah, it's allowable, it's just a YAML key. specify a string if you want.
|
data/bin/awshucks
ADDED
data/lib/awshucks.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
module Awshucks
|
2
|
+
|
3
|
+
ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__),".."))
|
4
|
+
LIB_DIR = File.join(ROOT_DIR,"lib").freeze
|
5
|
+
RESOURCE_DIR = File.join(ROOT_DIR,"resources").freeze
|
6
|
+
|
7
|
+
#
|
8
|
+
# Utility method to require all files ending in .rb in the directory
|
9
|
+
# with the same name as this file minus .rb
|
10
|
+
#
|
11
|
+
def require_all_libs_relative_to(fname)
|
12
|
+
prepend = File.basename(fname,".rb")
|
13
|
+
search_me = File.join(File.dirname(fname),prepend)
|
14
|
+
|
15
|
+
Dir.entries(search_me).each do |rb|
|
16
|
+
if File.extname(rb) == ".rb" then
|
17
|
+
require "#{prepend}/#{File.basename(rb,".rb")}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
module_function :require_all_libs_relative_to
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'rubygems'
|
26
|
+
require 'optparse'
|
27
|
+
require 'aws/s3'
|
28
|
+
|
29
|
+
Awshucks.require_all_libs_relative_to(__FILE__)
|
data/lib/awshucks/cli.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
module Awshucks
|
2
|
+
|
3
|
+
class CLI
|
4
|
+
|
5
|
+
def self.execute
|
6
|
+
new.execute(ARGV)
|
7
|
+
end
|
8
|
+
|
9
|
+
# parses the options and runs the command
|
10
|
+
def execute(args)
|
11
|
+
|
12
|
+
option_parser.parse!(args) # changes args!
|
13
|
+
|
14
|
+
if parsed_options.show_help || args.empty?
|
15
|
+
puts help_message
|
16
|
+
else
|
17
|
+
Command.parse_and_execute(args, config)
|
18
|
+
end
|
19
|
+
|
20
|
+
rescue CommandError => e
|
21
|
+
error(e)
|
22
|
+
end
|
23
|
+
|
24
|
+
def help_message
|
25
|
+
option_parser.to_s
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def error(msg)
|
31
|
+
$stderr.puts(msg)
|
32
|
+
exit(-1)
|
33
|
+
end
|
34
|
+
|
35
|
+
def option_parser
|
36
|
+
|
37
|
+
OptionParser.new do |opts|
|
38
|
+
opts.banner = "Usage: awshucks [options] <command> [command options]"
|
39
|
+
opts.separator "awshucks version #{Awshucks::VERSION}"
|
40
|
+
opts.separator "type 'awshucks help <subcommand'> for help with a specific command"
|
41
|
+
opts.separator "The default configuration file used is awshucks.yml in the current working directory."
|
42
|
+
opts.separator ""
|
43
|
+
opts.separator "Options:"
|
44
|
+
|
45
|
+
opts.on('-h', '--help', "Displays this help message") do
|
46
|
+
parsed_options.show_help = true
|
47
|
+
end
|
48
|
+
|
49
|
+
opts.on('-c', '--config-file FILE', "Set the awshucks config file to FILE") do |config_file|
|
50
|
+
parsed_options.config_file = config_file
|
51
|
+
end
|
52
|
+
|
53
|
+
opts.separator ""
|
54
|
+
opts.separator "Available Commands:"
|
55
|
+
|
56
|
+
Command.commands.keys.sort.each do |cmd|
|
57
|
+
opts.separator " " << Command.commands[cmd].command
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def parsed_options
|
64
|
+
@parsed_options ||= OpenStruct.new
|
65
|
+
end
|
66
|
+
|
67
|
+
def config
|
68
|
+
config_file = parsed_options.config_file || 'awshucks.yml'
|
69
|
+
@config ||= returning(Config.new(config_file)) do |config|
|
70
|
+
config.help_message = help_message
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module Awshucks
|
2
|
+
|
3
|
+
class CommandError < Exception; end
|
4
|
+
|
5
|
+
class ExecutionError < CommandError; end
|
6
|
+
|
7
|
+
class UnknownCommandError < CommandError
|
8
|
+
def initialize(msg)
|
9
|
+
super "Could not find command '#{msg}'"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
# wraps up command parsing functions. Command class has class-level accessor for registering commands.
|
14
|
+
class Command
|
15
|
+
|
16
|
+
##### class methods #####
|
17
|
+
|
18
|
+
class << self
|
19
|
+
|
20
|
+
def inherited(klass)
|
21
|
+
# save it for later, klass doesn't get fully defined until after this callback
|
22
|
+
inherited_commands << klass
|
23
|
+
end
|
24
|
+
|
25
|
+
def parse_and_execute(args, config)
|
26
|
+
key = args.shift
|
27
|
+
if key && commands[key] # be explicit about nil, no invalid command inheritance hijinks allowed
|
28
|
+
commands[key].execute(args, config)
|
29
|
+
else
|
30
|
+
raise UnknownCommandError, key
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def commands
|
35
|
+
@commands ||= returning({}) do |commands|
|
36
|
+
inherited_commands.each do |command|
|
37
|
+
commands[command.command] = command
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def option_string
|
43
|
+
new.option_parser.to_s # ehhh not a huge fan of this...
|
44
|
+
end
|
45
|
+
|
46
|
+
def execute(args, config)
|
47
|
+
new.execute(args, config)
|
48
|
+
end
|
49
|
+
|
50
|
+
attr_accessor :command
|
51
|
+
attr_accessor :usage
|
52
|
+
attr_accessor :description
|
53
|
+
|
54
|
+
#######
|
55
|
+
private
|
56
|
+
#######
|
57
|
+
|
58
|
+
def inherited_commands
|
59
|
+
@inherited ||= []
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
|
64
|
+
##### instance methods ####
|
65
|
+
|
66
|
+
def execute(args, config)
|
67
|
+
raise CommandError, "must implement the execute method!"
|
68
|
+
end
|
69
|
+
|
70
|
+
def option_parser
|
71
|
+
@option_parser ||= OptionParser.new do |opts|
|
72
|
+
opts.banner = self.class.usage
|
73
|
+
opts.separator self.class.description
|
74
|
+
opts.separator ""
|
75
|
+
if respond_to?(:custom_options)
|
76
|
+
opts.separator "Options:"
|
77
|
+
custom_options(opts)
|
78
|
+
else
|
79
|
+
opts.separator "No options."
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
# define the method custom_options(opts) if options are required for a command
|
85
|
+
|
86
|
+
#######
|
87
|
+
private
|
88
|
+
#######
|
89
|
+
|
90
|
+
# for use in the option definitions
|
91
|
+
def parsed_options
|
92
|
+
@parsed_options ||= OpenStruct.new
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module Awshucks
|
2
|
+
|
3
|
+
class BackupCommand < Command
|
4
|
+
self.command = 'backup'
|
5
|
+
self.usage = 'backup [config name]'
|
6
|
+
self.description = "Backs up all locations specified in the config file. " +
|
7
|
+
"If you specify a config name, awshucks will back up only the files specified in that config."
|
8
|
+
|
9
|
+
def execute(args, config)
|
10
|
+
@config = config
|
11
|
+
list = []
|
12
|
+
if args.empty?
|
13
|
+
list += config.backups
|
14
|
+
else
|
15
|
+
list << config.backup(args.first)
|
16
|
+
end
|
17
|
+
|
18
|
+
list.each {|b| backup(b) }
|
19
|
+
|
20
|
+
rescue UnknownBackupError
|
21
|
+
$stderr.puts "Unknown backup: #{args.first}"
|
22
|
+
end
|
23
|
+
|
24
|
+
#######
|
25
|
+
private
|
26
|
+
#######
|
27
|
+
|
28
|
+
attr_reader :config
|
29
|
+
|
30
|
+
def backup(backup_info)
|
31
|
+
|
32
|
+
puts "backing up #{backup_info.location}:"
|
33
|
+
|
34
|
+
file_store = FileStore.new(config.connection, config.bucket, backup_info.name)
|
35
|
+
|
36
|
+
base_path = File.expand_path(backup_info.location)
|
37
|
+
|
38
|
+
begin
|
39
|
+
Scanner.each_file(base_path) do |file_info|
|
40
|
+
print " #{file_info.filename}: "
|
41
|
+
full_path = File.join(base_path, file_info.filename)
|
42
|
+
if file_store.different_from?(file_info)
|
43
|
+
print "changed, uploading (#{file_info.size} bytes)..."
|
44
|
+
file_store.store(file_info)
|
45
|
+
puts "done."
|
46
|
+
else
|
47
|
+
puts "unchanged."
|
48
|
+
end
|
49
|
+
end
|
50
|
+
ensure
|
51
|
+
puts "saving metadata cache..."
|
52
|
+
file_store.save_cache
|
53
|
+
end
|
54
|
+
|
55
|
+
end # backup
|
56
|
+
|
57
|
+
end # class
|
58
|
+
|
59
|
+
end
|