awshucks 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES +4 -0
- data/LICENSE +19 -0
- data/README +32 -0
- data/Rakefile +14 -0
- data/TODO +143 -0
- data/bin/awshucks +10 -0
- data/lib/awshucks.rb +29 -0
- data/lib/awshucks/cli.rb +76 -0
- data/lib/awshucks/command.rb +98 -0
- data/lib/awshucks/commands.rb +3 -0
- data/lib/awshucks/commands/backup.rb +59 -0
- data/lib/awshucks/commands/backups.rb +25 -0
- data/lib/awshucks/commands/help.rb +22 -0
- data/lib/awshucks/commands/list.rb +39 -0
- data/lib/awshucks/commands/new_config.rb +36 -0
- data/lib/awshucks/commands/reset_metadata_cache.rb +38 -0
- data/lib/awshucks/commands/restore.rb +54 -0
- data/lib/awshucks/config.rb +83 -0
- data/lib/awshucks/ext.rb +23 -0
- data/lib/awshucks/file_info.rb +23 -0
- data/lib/awshucks/file_store.rb +111 -0
- data/lib/awshucks/gemspec.rb +48 -0
- data/lib/awshucks/scanner.rb +58 -0
- data/lib/awshucks/specification.rb +128 -0
- data/lib/awshucks/version.rb +18 -0
- data/resources/awshucks.yml +21 -0
- data/spec/awshucks_spec.rb +7 -0
- data/spec/cli_spec.rb +130 -0
- data/spec/command_spec.rb +111 -0
- data/spec/commands/backup_spec.rb +164 -0
- data/spec/commands/backups_spec.rb +41 -0
- data/spec/commands/help_spec.rb +42 -0
- data/spec/commands/list_spec.rb +77 -0
- data/spec/commands/new_config_spec.rb +102 -0
- data/spec/commands/reset_metadata_cache_spec.rb +93 -0
- data/spec/commands/restore_spec.rb +219 -0
- data/spec/config_spec.rb +152 -0
- data/spec/ext_spec.rb +28 -0
- data/spec/file_info_spec.rb +45 -0
- data/spec/file_store_spec.rb +352 -0
- data/spec/scanner_spec.rb +106 -0
- data/spec/spec_helper.rb +36 -0
- data/spec/specification_spec.rb +41 -0
- metadata +121 -0
@@ -0,0 +1,219 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__),"..","spec_helper.rb")
|
2
|
+
|
3
|
+
describe Awshucks::RestoreCommand do
|
4
|
+
it "has 'restore' as the command" do
|
5
|
+
Awshucks::RestoreCommand.command.should == 'restore'
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
describe Awshucks::RestoreCommand, '#execute, with no backup configuration specified' do
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
@config = mock('config')
|
13
|
+
@restore = Awshucks::RestoreCommand.new
|
14
|
+
@old_stderr, $stderr = $stderr, StringIO.new
|
15
|
+
end
|
16
|
+
|
17
|
+
after(:each) do
|
18
|
+
$stderr = @old_stderr
|
19
|
+
end
|
20
|
+
|
21
|
+
it "prints out an error" do
|
22
|
+
@restore.execute([], @config)
|
23
|
+
$stderr.string.should =~ /specify a backup/i
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
describe Awshucks::RestoreCommand, '#execute, with just a backup configuration specified' do
|
29
|
+
|
30
|
+
before(:each) do
|
31
|
+
@connection = mock('connection')
|
32
|
+
|
33
|
+
@backup = mock('backup configuration')
|
34
|
+
@backup.stub!(:name).and_return('testbackup')
|
35
|
+
@backup.stub!(:location).and_return('/test/location')
|
36
|
+
|
37
|
+
@config = mock('config')
|
38
|
+
@config.stub!(:connection).and_return(@connection)
|
39
|
+
@config.stub!(:backup).and_return(@backup)
|
40
|
+
|
41
|
+
@config.stub!(:bucket).and_return('testbucket')
|
42
|
+
@restore = Awshucks::RestoreCommand.new
|
43
|
+
@old_stderr, $stderr = $stderr, StringIO.new
|
44
|
+
@old_stdout, $stdout = $stdout, StringIO.new
|
45
|
+
|
46
|
+
@file_store = mock(Awshucks::FileStore)
|
47
|
+
end
|
48
|
+
|
49
|
+
after(:each) do
|
50
|
+
$stderr = @old_stderr
|
51
|
+
$stdout = @old_stdout
|
52
|
+
end
|
53
|
+
|
54
|
+
it "instantiates a FileStore object for the backup config" do
|
55
|
+
Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
|
56
|
+
@file_store.stub!(:each_file)
|
57
|
+
@restore.execute(['testbackup'], @config)
|
58
|
+
end
|
59
|
+
|
60
|
+
it "iterates through each file in the file store and calls restore on each entry" do
|
61
|
+
Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
|
62
|
+
@file_store.should_receive(:each_file).and_yield('file')
|
63
|
+
@file_store.should_receive(:restore).with('file', '/test/location/file')
|
64
|
+
|
65
|
+
@restore.execute(['testbackup'], @config)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "prints a warning if the backup config isn't found" do
|
69
|
+
@config.stub!(:backup).and_raise(Awshucks::UnknownBackupError.new('foo'))
|
70
|
+
@restore.execute(['foo'], @config)
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
74
|
+
|
75
|
+
describe Awshucks::RestoreCommand, '#execute, with a backup configuration and a specific file' do
|
76
|
+
|
77
|
+
before(:each) do
|
78
|
+
@connection = mock('connection')
|
79
|
+
|
80
|
+
@backup = mock('backup configuration')
|
81
|
+
@backup.stub!(:name).and_return('testbackup')
|
82
|
+
@backup.stub!(:location).and_return('/test/location')
|
83
|
+
|
84
|
+
@config = mock('config')
|
85
|
+
@config.stub!(:connection).and_return(@connection)
|
86
|
+
@config.stub!(:backup).and_return(@backup)
|
87
|
+
|
88
|
+
@config.stub!(:bucket).and_return('testbucket')
|
89
|
+
@restore = Awshucks::RestoreCommand.new
|
90
|
+
|
91
|
+
@old_stdout, $stdout = $stdout, StringIO.new
|
92
|
+
|
93
|
+
@file_store = mock(Awshucks::FileStore)
|
94
|
+
end
|
95
|
+
|
96
|
+
after(:each) do
|
97
|
+
$stdout = @old_stdout
|
98
|
+
end
|
99
|
+
|
100
|
+
it "iterates through each file in the file store and calls restore on each entry" do
|
101
|
+
Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
|
102
|
+
class << @file_store
|
103
|
+
def each_file
|
104
|
+
yield 'other_file'
|
105
|
+
yield 'specific_file'
|
106
|
+
yield 'another/file'
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
@file_store.should_receive(:restore).with('specific_file', '/test/location/specific_file')
|
111
|
+
|
112
|
+
@restore.execute(['testbackup', 'specific_file'], @config)
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
117
|
+
describe Awshucks::RestoreCommand, '#execute, with a backup configuration and a pattern' do
|
118
|
+
|
119
|
+
before(:each) do
|
120
|
+
@connection = mock('connection')
|
121
|
+
|
122
|
+
@backup = mock('backup configuration')
|
123
|
+
@backup.stub!(:name).and_return('testbackup')
|
124
|
+
@backup.stub!(:location).and_return('/test/location')
|
125
|
+
|
126
|
+
@config = mock('config')
|
127
|
+
@config.stub!(:connection).and_return(@connection)
|
128
|
+
@config.stub!(:backup).and_return(@backup)
|
129
|
+
|
130
|
+
@config.stub!(:bucket).and_return('testbucket')
|
131
|
+
@restore = Awshucks::RestoreCommand.new
|
132
|
+
|
133
|
+
@old_stdout, $stdout = $stdout, StringIO.new
|
134
|
+
|
135
|
+
@file_store = mock(Awshucks::FileStore)
|
136
|
+
end
|
137
|
+
|
138
|
+
after(:each) do
|
139
|
+
$stdout = @old_stdout
|
140
|
+
end
|
141
|
+
|
142
|
+
it "restores all files when the pattern is '*'" do
|
143
|
+
Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
|
144
|
+
class << @file_store
|
145
|
+
def each_file
|
146
|
+
yield 'some_file'
|
147
|
+
yield 'another/file'
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
@file_store.should_receive(:restore).with('some_file', '/test/location/some_file')
|
152
|
+
@file_store.should_receive(:restore).with('another/file', '/test/location/another/file')
|
153
|
+
|
154
|
+
@restore.execute(['testbackup', '*'], @config)
|
155
|
+
end
|
156
|
+
|
157
|
+
it "restores only the files matching the pattern" do
|
158
|
+
Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
|
159
|
+
class << @file_store
|
160
|
+
def each_file
|
161
|
+
yield 'some_file'
|
162
|
+
yield 'sub/file'
|
163
|
+
yield 'foo/bar'
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
@file_store.should_receive(:restore).with('some_file', '/test/location/some_file')
|
168
|
+
@file_store.should_receive(:restore).with('foo/bar', '/test/location/foo/bar')
|
169
|
+
|
170
|
+
@restore.execute(['testbackup', '*o*'], @config)
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
|
175
|
+
describe Awshucks::RestoreCommand, '#execute, with a backup, pattern, and a target directory' do
|
176
|
+
|
177
|
+
before(:each) do
|
178
|
+
@connection = mock('connection')
|
179
|
+
|
180
|
+
@backup = mock('backup configuration')
|
181
|
+
@backup.stub!(:name).and_return('testbackup')
|
182
|
+
@backup.stub!(:location).and_return('/test/location')
|
183
|
+
|
184
|
+
@config = mock('config')
|
185
|
+
@config.stub!(:connection).and_return(@connection)
|
186
|
+
@config.stub!(:backup).and_return(@backup)
|
187
|
+
|
188
|
+
@config.stub!(:bucket).and_return('testbucket')
|
189
|
+
@restore = Awshucks::RestoreCommand.new
|
190
|
+
|
191
|
+
@old_stdout, $stdout = $stdout, StringIO.new
|
192
|
+
|
193
|
+
@file_store = mock(Awshucks::FileStore)
|
194
|
+
end
|
195
|
+
|
196
|
+
after(:each) do
|
197
|
+
$stdout = @old_stdout
|
198
|
+
end
|
199
|
+
|
200
|
+
it "calls restore using the target directory as the base directory for the restore" do
|
201
|
+
Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
|
202
|
+
@file_store.should_receive(:each_file).and_yield('test/file')
|
203
|
+
|
204
|
+
@file_store.should_receive(:restore).with('test/file', '/another/place/test/file')
|
205
|
+
|
206
|
+
@restore.execute(['testbackup', '*', '/another/place'], @config)
|
207
|
+
end
|
208
|
+
|
209
|
+
it "calls restore with the target being a fully expanded path" do
|
210
|
+
Awshucks::FileStore.stub!(:new).and_return(@file_store)
|
211
|
+
@file_store.stub!(:each_file).and_yield('test/file')
|
212
|
+
|
213
|
+
@file_store.should_receive(:restore).with('test/file', '/full/path/another/place/test/file')
|
214
|
+
|
215
|
+
File.should_receive(:expand_path).with('another/place').and_return('/full/path/another/place')
|
216
|
+
@restore.execute(['testbackup', '*', 'another/place'], @config)
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
data/spec/config_spec.rb
ADDED
@@ -0,0 +1,152 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__),"spec_helper.rb")
|
2
|
+
|
3
|
+
describe Awshucks::Config do
|
4
|
+
|
5
|
+
before(:each) do
|
6
|
+
@old_stderr, $stderr = $stderr, StringIO.new
|
7
|
+
end
|
8
|
+
|
9
|
+
after(:each) do
|
10
|
+
$stderr = @old_stderr
|
11
|
+
end
|
12
|
+
|
13
|
+
it "accepts a filename as a parameter to initialize" do
|
14
|
+
config = Awshucks::Config.new( fixture_file('awshucks.yml') )
|
15
|
+
end
|
16
|
+
|
17
|
+
it "lazily loads the yaml file" do
|
18
|
+
config = Awshucks::Config.new( fixture_file('awshucks.yml') )
|
19
|
+
# YAML.should_receive(:load).and_return({})
|
20
|
+
# config.send(:config) # force the lazy load
|
21
|
+
config.connection.should == {
|
22
|
+
:access_key_id => "access key",
|
23
|
+
:secret_access_key => 'secret access key',
|
24
|
+
:persistent => false,
|
25
|
+
:use_ssl => true
|
26
|
+
}
|
27
|
+
end
|
28
|
+
|
29
|
+
it "prints an error and exits if the config file wasn't found" do
|
30
|
+
config = Awshucks::Config.new( fixture_file('awshucks.yml') )
|
31
|
+
YAML.should_receive(:load).and_raise(Errno::ENOENT)
|
32
|
+
lambda { config.send(:config) }.should raise_error(SystemExit)
|
33
|
+
$stderr.string.should =~ /not found/
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
# describe Awshucks::Config, '.load' do
|
39
|
+
#
|
40
|
+
# before(:each) do
|
41
|
+
# @config = Awshucks::Config.load(fixture_file('awshucks.yml'))
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# it "should load the specified file as YAML" do
|
45
|
+
# @config.connection.should == {
|
46
|
+
# :access_key_id => "access key",
|
47
|
+
# :secret_access_key => 'secret access key',
|
48
|
+
# :persistent => false,
|
49
|
+
# :use_ssl => true
|
50
|
+
# }
|
51
|
+
# end
|
52
|
+
#
|
53
|
+
# it "should return a Config object" do
|
54
|
+
# @config.should be_instance_of(Awshucks::Config)
|
55
|
+
# end
|
56
|
+
# end
|
57
|
+
|
58
|
+
describe Awshucks::Config, '#connection' do
|
59
|
+
|
60
|
+
it "should return the connection info specified in the config file, appropriate for the AWS library" do
|
61
|
+
config = Awshucks::Config.new(fixture_file('awshucks.yml'))
|
62
|
+
config.connection.should == {
|
63
|
+
:access_key_id => "access key",
|
64
|
+
:secret_access_key => 'secret access key',
|
65
|
+
:persistent => false,
|
66
|
+
:use_ssl => true
|
67
|
+
}
|
68
|
+
end
|
69
|
+
|
70
|
+
Awshucks::Config::REQUIRED_CONNECTION_KEYS.each do |key|
|
71
|
+
it "should raise an exception if the config file is missing the #{key} connection option" do
|
72
|
+
data = YAML.load(fixture_file('awshucks.yml'))
|
73
|
+
data.delete(key)
|
74
|
+
File.stub!(:open)
|
75
|
+
YAML.should_receive(:load).and_return(data)
|
76
|
+
config = Awshucks::Config.new('lol')
|
77
|
+
lambda { config.connection }.should raise_error(Awshucks::RequiredConfigError)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
|
83
|
+
describe Awshucks::Config, '#bucket' do
|
84
|
+
before(:each) do
|
85
|
+
@config = Awshucks::Config.new(fixture_file('awshucks.yml'))
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should return the configured bucket name" do
|
89
|
+
@config.bucket.should == "testbucket"
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
describe Awshucks::Config, '#backup' do
|
95
|
+
before(:each) do
|
96
|
+
@config = Awshucks::Config.new(fixture_file('awshucks.yml'))
|
97
|
+
end
|
98
|
+
|
99
|
+
it "should return the information for a specific backup location" do
|
100
|
+
@config.backup('test_backup').should_not be_nil
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should return an object with a location attribute" do
|
104
|
+
@config.backup('test_backup').location.should == '/primary/backup'
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should return an object with a name attribute" do
|
108
|
+
@config.backup('test_backup').name.should == 'test_backup'
|
109
|
+
end
|
110
|
+
|
111
|
+
Awshucks::Config::REQUIRED_CONNECTION_KEYS.each do |word|
|
112
|
+
it "should raise an exception if the backup name is the reserved config name '#{word}'" do
|
113
|
+
lambda { @config.backup(word) }.should raise_error(Awshucks::UnknownBackupError)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
Awshucks::Config::REQUIRED_BACKUP_CONFIG_KEYS.each do |key|
|
118
|
+
it "should raise an exception if the backup is missing a #{key}" do
|
119
|
+
lambda { @config.backup('invalid_backup') }.should raise_error(Awshucks::RequiredConfigError)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
end
|
124
|
+
|
125
|
+
describe Awshucks::Config, '#backups' do
|
126
|
+
before(:each) do
|
127
|
+
@config = Awshucks::Config.new(fixture_file('awshucks.yml'))
|
128
|
+
end
|
129
|
+
|
130
|
+
it "should return the list of backup configurations in the config" do
|
131
|
+
@config.backups.should have_at_least(1).item
|
132
|
+
end
|
133
|
+
|
134
|
+
it "should return backup configuration items" do
|
135
|
+
@config.backups.first.name.should == 'test_backup'
|
136
|
+
end
|
137
|
+
|
138
|
+
Awshucks::Config::RESERVED_KEYS.each do |key|
|
139
|
+
it "should not include the reserved key #{key}" do
|
140
|
+
@config.backups.map {|b| b.name}.should_not include(key)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
|
146
|
+
describe Awshucks::Config do
|
147
|
+
it "has a help_message attribute" do
|
148
|
+
@config = Awshucks::Config.new(fixture_file('awshucks.yml'))
|
149
|
+
@config.help_message = 'help message'
|
150
|
+
@config.help_message.should == 'help message'
|
151
|
+
end
|
152
|
+
end
|
data/spec/ext_spec.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__),"spec_helper.rb")
|
2
|
+
|
3
|
+
describe Object, '#returning' do
|
4
|
+
it "yields the provided argument, then returns it" do
|
5
|
+
yielded = nil
|
6
|
+
returning(:foo) do |obj|
|
7
|
+
yielded = obj
|
8
|
+
end.should == :foo
|
9
|
+
yielded.should == :foo
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
describe Object, '#in?' do
|
14
|
+
it "calls include? on the provided enumerable" do
|
15
|
+
collection = mock('collection')
|
16
|
+
item = mock('item')
|
17
|
+
collection.should_receive(:include?).with(item).and_return(true)
|
18
|
+
item.in?(collection).should be_true
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe File, '.md5sum' do
|
23
|
+
it "calls read on the provided filename and computes the MD5 hex digest" do
|
24
|
+
sum = Digest::MD5.hexdigest('lol')
|
25
|
+
File.should_receive('read').with('pretend/file').and_return('lol')
|
26
|
+
File.md5sum('pretend/file').should == sum
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__),"spec_helper.rb")
|
2
|
+
|
3
|
+
describe Awshucks::FileInfo do
|
4
|
+
|
5
|
+
before(:each) do
|
6
|
+
@now = Time.now
|
7
|
+
@stat = mock('file stat instance')
|
8
|
+
@stat.stub!(:mtime).and_return(@now)
|
9
|
+
@stat.stub!(:size).and_return(12345)
|
10
|
+
@info = Awshucks::FileInfo.new('file', '/path/to/file', @stat)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "is initialized with a relative filename, a full path, and a File::Stat object" do
|
14
|
+
@info.should be_instance_of(Awshucks::FileInfo)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "has a filename" do
|
18
|
+
@info.filename.should == 'file'
|
19
|
+
end
|
20
|
+
|
21
|
+
it "has a full filename path" do
|
22
|
+
@info.full_path.should == '/path/to/file'
|
23
|
+
end
|
24
|
+
|
25
|
+
it "has an mtime in UTC" do
|
26
|
+
@info.mtime.should be_kind_of(Time)
|
27
|
+
@info.mtime.to_s.should == @now.gmtime.to_s
|
28
|
+
end
|
29
|
+
|
30
|
+
it "has a lazily-evaluated md5sum" do
|
31
|
+
File.should_receive(:md5sum).with('/path/to/file').and_return('placeholder sum')
|
32
|
+
@info.md5sum.should == 'placeholder sum'
|
33
|
+
end
|
34
|
+
|
35
|
+
it "returns an IO object when asked for the data" do
|
36
|
+
io = mock('pretend io')
|
37
|
+
File.should_receive(:open).and_return(io)
|
38
|
+
@info.data.should == io
|
39
|
+
end
|
40
|
+
|
41
|
+
it "has a size (in bytes)" do
|
42
|
+
@info.size.should == 12345
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
@@ -0,0 +1,352 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__),"spec_helper.rb")
|
2
|
+
|
3
|
+
describe Awshucks::FileStore do
|
4
|
+
|
5
|
+
before(:each) do
|
6
|
+
@connection = mock('connection')
|
7
|
+
AWS::S3::Base.stub!(:establish_connection!)
|
8
|
+
end
|
9
|
+
|
10
|
+
it "accepts a connection info hash, bucket name, and a backup prefix as parameters to initialize" do
|
11
|
+
Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix').should be_instance_of(Awshucks::FileStore)
|
12
|
+
end
|
13
|
+
|
14
|
+
it "establishes a connection to S3 using the config when initialized" do
|
15
|
+
AWS::S3::Base.should_receive(:establish_connection!).with(@connection)
|
16
|
+
fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
|
21
|
+
describe Awshucks::FileStore, '#different_from? with a FileInfo object' do
|
22
|
+
|
23
|
+
before(:each) do
|
24
|
+
|
25
|
+
@bucket = mock('bucket')
|
26
|
+
@connection = mock('connection')
|
27
|
+
AWS::S3::Base.stub!(:establish_connection!)
|
28
|
+
AWS::S3::Bucket.stub!(:find).and_return(@bucket)
|
29
|
+
|
30
|
+
# for the metadata cache:
|
31
|
+
@now = Time.now.gmtime
|
32
|
+
@sum = "7589253b2c5f744411e43be22eb45b71" # "lol what"
|
33
|
+
@sum2 = "0d705691a3dddd72b11f5e9b7d61a306" # "no wai"
|
34
|
+
@meta = mock('metadata')
|
35
|
+
@meta.stub!(:value).and_return("---\n{}") # empty hash
|
36
|
+
@bucket.should_receive(:[]).any_number_of_times.with('testprefix_metadata.yml').and_return(@meta)
|
37
|
+
|
38
|
+
# for a pretend file stored on S3:
|
39
|
+
@file = mock('example stored s3 file')
|
40
|
+
@file.stub!(:key).and_return('testprefix/test/file')
|
41
|
+
|
42
|
+
@info = mock(Awshucks::FileInfo)
|
43
|
+
@info.stub!(:filename).and_return('test/file')
|
44
|
+
# @info.stub!(:full_path).and_return('/path/to/file')
|
45
|
+
|
46
|
+
@fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
it "returns true if the file isn't stored in the file store already" do
|
51
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([])
|
52
|
+
@fs.different_from?(@info).should be_true
|
53
|
+
end
|
54
|
+
|
55
|
+
it "returns false if the file's metadata mtime (in UTC!) matches" do
|
56
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
|
57
|
+
@file.stub!(:metadata).and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
|
58
|
+
@info.should_receive(:mtime).at_least(1).times.and_return(@now)
|
59
|
+
@info.stub!(:md5sum).and_return(@sum)
|
60
|
+
@fs.different_from?(@info).should be_false
|
61
|
+
end
|
62
|
+
|
63
|
+
it "returns false if the file's mtime is different but the md5sum still matches" do
|
64
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
|
65
|
+
|
66
|
+
@file.stub!(:metadata).and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
|
67
|
+
@info.should_receive(:mtime).any_number_of_times.and_return(@now+5)
|
68
|
+
@info.should_receive(:md5sum).and_return(@sum)
|
69
|
+
@fs.different_from?(@info).should be_false
|
70
|
+
end
|
71
|
+
|
72
|
+
it "updates the metadata cache with the new time if the file's mtime is different but md5sum is not" do
|
73
|
+
@meta.should_receive(:value).and_return(
|
74
|
+
{'test/file' => {'mtime' => @now, 'md5sum' => @sum}}.to_yaml
|
75
|
+
)
|
76
|
+
@info.should_receive(:mtime).twice.and_return(@now+5)
|
77
|
+
@info.should_receive(:md5sum).and_return(@sum)
|
78
|
+
@fs.different_from?(@info).should be_false
|
79
|
+
@fs.send(:metadata)['test/file']['mtime'].should == (@now+5).gmtime.to_s
|
80
|
+
end
|
81
|
+
|
82
|
+
it "returns true if the file's mtime != the provided mtime and the md5sum doesn't match" do
|
83
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
|
84
|
+
|
85
|
+
@file.stub!(:metadata).and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
|
86
|
+
@info.should_receive(:mtime).and_return(@now+5)
|
87
|
+
@info.should_receive(:md5sum).and_return(@sum2)
|
88
|
+
|
89
|
+
@fs.different_from?(@info).should be_true
|
90
|
+
end
|
91
|
+
|
92
|
+
it "preferentially uses the metadata cache to retrieve file information" do
|
93
|
+
@meta.should_receive(:value).and_return(
|
94
|
+
{'test/file' => {'mtime' => @now, 'md5sum' => @sum}}.to_yaml
|
95
|
+
)
|
96
|
+
|
97
|
+
@info.should_receive(:mtime).and_return(@now+5)
|
98
|
+
@info.should_receive(:md5sum).and_return(@sum2)
|
99
|
+
|
100
|
+
@fs.different_from?(@info).should be_true
|
101
|
+
end
|
102
|
+
|
103
|
+
it "falls back to the stored file on s3 for metadata if it's not in the cache" do
|
104
|
+
# the metadata cache is empty...
|
105
|
+
@file.should_receive(:metadata).at_least(1).times.and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
|
106
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
|
107
|
+
|
108
|
+
@info.should_receive(:mtime).and_return(@now+5)
|
109
|
+
@info.should_receive(:md5sum).and_return(@sum2)
|
110
|
+
|
111
|
+
@fs.different_from?(@info).should be_true
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
|
116
|
+
describe Awshucks::FileStore, '#each_file' do
|
117
|
+
|
118
|
+
before(:each) do
|
119
|
+
AWS::S3::Base.stub!(:establish_connection!)
|
120
|
+
@bucket = mock('bucket')
|
121
|
+
@bucket.stub!(:name).and_return('testbucket')
|
122
|
+
@bucket.stub!(:[])
|
123
|
+
|
124
|
+
AWS::S3::Bucket.stub!(:find).and_return(@bucket)
|
125
|
+
|
126
|
+
@fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
|
127
|
+
end
|
128
|
+
|
129
|
+
it "yields each filename stored in the bucket matching the configured prefix, with the prefix removed" do
|
130
|
+
@fileone = mock('file one')
|
131
|
+
@fileone.stub!(:key).and_return('testprefix/some/file')
|
132
|
+
@filetwo = mock('file two')
|
133
|
+
@filetwo.stub!(:key).and_return('testprefix/another')
|
134
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@fileone, @filetwo])
|
135
|
+
|
136
|
+
yielded = []
|
137
|
+
@fs.each_file { |filename| yielded << filename }
|
138
|
+
yielded.should == %w(some/file another)
|
139
|
+
end
|
140
|
+
|
141
|
+
it "does not yield the metadata file" do
|
142
|
+
@meta = mock('metadata')
|
143
|
+
@meta.stub!(:key).and_return('testprefix_metadata.yml')
|
144
|
+
@fileone = mock('file one')
|
145
|
+
@fileone.stub!(:key).and_return('testprefix/some/file')
|
146
|
+
@bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@fileone, @meta])
|
147
|
+
|
148
|
+
yielded = []
|
149
|
+
@fs.each_file { |filename| yielded << filename }
|
150
|
+
yielded.should == %w(some/file)
|
151
|
+
end
|
152
|
+
|
153
|
+
end
|
154
|
+
|
155
|
+
describe Awshucks::FileStore, '#store' do
|
156
|
+
|
157
|
+
before(:each) do
|
158
|
+
AWS::S3::Base.stub!(:establish_connection!)
|
159
|
+
|
160
|
+
@bucket = mock('bucket')
|
161
|
+
@bucket.stub!(:name).and_return('testbucket')
|
162
|
+
@bucket.stub!(:[])
|
163
|
+
|
164
|
+
AWS::S3::Bucket.stub!(:find).and_return(@bucket)
|
165
|
+
|
166
|
+
@response = mock('s3 response object')
|
167
|
+
@response.stub!(:success?).and_return true
|
168
|
+
|
169
|
+
@gmtime = Time.now.gmtime
|
170
|
+
@sum = 'pretend md5 sum'
|
171
|
+
|
172
|
+
@info = mock('fileinfo')
|
173
|
+
@info.stub!(:filename).and_return('test/file')
|
174
|
+
@info.stub!(:mtime).and_return(@gmtime)
|
175
|
+
@info.stub!(:data).and_return('data')
|
176
|
+
@info.stub!(:md5sum).and_return(@sum)
|
177
|
+
|
178
|
+
::AWS::S3::S3Object.should_receive(:store).with('testprefix/test/file', 'data', 'testbucket', {
|
179
|
+
'x-amz-meta-mtime' => @gmtime.to_s, 'x-amz-meta-md5sum' => @sum
|
180
|
+
}).and_return(@response)
|
181
|
+
|
182
|
+
@fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
|
183
|
+
end
|
184
|
+
|
185
|
+
it "stores the provided file data, including the modified time as metadata" do
|
186
|
+
@fs.store(@info)
|
187
|
+
end
|
188
|
+
|
189
|
+
it "includes the stored file in the metadata cache" do
|
190
|
+
@fs.store(@info)
|
191
|
+
@fs.send(:metadata).should == {'test/file' => {'mtime' => @gmtime.to_s, 'md5sum' => @sum}}
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
|
196
|
+
describe Awshucks::FileStore, '#restore' do
|
197
|
+
|
198
|
+
before(:each) do
|
199
|
+
AWS::S3::Base.stub!(:establish_connection!)
|
200
|
+
@bucket = mock('bucket')
|
201
|
+
@bucket.stub!(:name).and_return('testbucket')
|
202
|
+
@bucket.stub!(:[])
|
203
|
+
|
204
|
+
@sum = "imaginary md5 sum"
|
205
|
+
|
206
|
+
@meta = mock('metadata object')
|
207
|
+
@meta.stub!(:key).and_return('testprefix_metadata.yml')
|
208
|
+
@meta.stub!(:value).and_return(
|
209
|
+
{'some/file' => {'mtime' => Time.now.gmtime.to_s, 'md5sum' => @sum}}.to_yaml
|
210
|
+
)
|
211
|
+
|
212
|
+
@bucket.should_receive(:[]).with('testprefix_metadata.yml').any_number_of_times.and_return(@meta)
|
213
|
+
|
214
|
+
AWS::S3::Bucket.stub!(:find).and_return(@bucket)
|
215
|
+
|
216
|
+
@file = mock('pretend s3 file')
|
217
|
+
@file.stub!(:key).and_return('testprefix/some/file')
|
218
|
+
@file.stub!(:value).and_yield('value')
|
219
|
+
|
220
|
+
# @bucket.stub!(:objects).and_return([@file, @meta])
|
221
|
+
|
222
|
+
@filehandle = mock('filehandle')
|
223
|
+
|
224
|
+
FileUtils.stub!(:mkdir_p)
|
225
|
+
File.stub!(:md5sum).and_return(@sum)
|
226
|
+
|
227
|
+
@fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
|
228
|
+
@old_stderr, $stderr = $stderr, StringIO.new
|
229
|
+
end
|
230
|
+
|
231
|
+
after(:each) do
|
232
|
+
$stderr = @old_stderr
|
233
|
+
end
|
234
|
+
|
235
|
+
it "raises an exception if the specified file doesn't exist in the bucket" do
|
236
|
+
@bucket.should_receive(:[]).with('testprefix/some/file').and_return(nil)
|
237
|
+
lambda {@fs.restore('some/file', 'foo/bar/some/file') }.should raise_error(Awshucks::UnknownFileError)
|
238
|
+
end
|
239
|
+
|
240
|
+
it "restores the specified file to the specified location" do
|
241
|
+
@bucket.should_receive(:[]).with('testprefix/some/file').and_return(@file)
|
242
|
+
File.should_receive(:open).with('/absolute/location/some/file', 'w').and_yield(@filehandle)
|
243
|
+
@filehandle.should_receive(:write).with('value')
|
244
|
+
@fs.restore('some/file', '/absolute/location/some/file')
|
245
|
+
end
|
246
|
+
|
247
|
+
it "creates the directory tree necessary to restore the file" do
|
248
|
+
@bucket.should_receive(:[]).with('testprefix/some/file').and_return(@file)
|
249
|
+
File.stub!(:open).and_yield(@filehandle)
|
250
|
+
@filehandle.stub!(:write)
|
251
|
+
|
252
|
+
FileUtils.should_receive(:mkdir_p).with('/absolute/location/some')
|
253
|
+
@fs.restore('some/file', '/absolute/location/some/file')
|
254
|
+
end
|
255
|
+
|
256
|
+
it "prints a warning if the md5sum of the written file doesn't match the stored metadata" do
|
257
|
+
@bucket.should_receive(:[]).with('testprefix/some/file').and_return(@file)
|
258
|
+
File.stub!(:open).and_yield(@filehandle)
|
259
|
+
@filehandle.stub!(:write)
|
260
|
+
|
261
|
+
File.should_receive(:md5sum).with('/absolute/location/some/file').and_return('different sum')
|
262
|
+
@fs.restore('some/file', '/absolute/location/some/file')
|
263
|
+
|
264
|
+
$stderr.string.should =~ /warning.*md5/i
|
265
|
+
end
|
266
|
+
|
267
|
+
end
|
268
|
+
|
269
|
+
describe Awshucks::FileStore, '#save_cache' do
|
270
|
+
|
271
|
+
before(:each) do
|
272
|
+
@bucket = mock('bucket')
|
273
|
+
@bucket.stub!(:name).and_return('testbucket')
|
274
|
+
@connection = mock('connection')
|
275
|
+
AWS::S3::Base.should_receive(:establish_connection!).with(@connection)
|
276
|
+
@fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
|
277
|
+
@fs.stub!(:bucket).and_return(@bucket)
|
278
|
+
@meta = mock('metadata')
|
279
|
+
@meta.stub!(:value).and_return("---\nfoo: bar")
|
280
|
+
@meta.stub!(:to_yaml).and_return("--- \nfoo: bar\n")
|
281
|
+
end
|
282
|
+
|
283
|
+
it "stores the metadata file on S3" do
|
284
|
+
@bucket.stub!(:[]).and_return(@meta)
|
285
|
+
AWS::S3::S3Object.should_receive(:store).with('testprefix_metadata.yml', "--- \nfoo: bar\n", 'testbucket')
|
286
|
+
@fs.save_cache
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
describe Awshucks::FileStore, '#reset_cache' do
|
291
|
+
|
292
|
+
before(:each) do
|
293
|
+
AWS::S3::Base.stub!(:establish_connection!)
|
294
|
+
@bucket = mock('bucket')
|
295
|
+
@bucket.stub!(:name).and_return('testbucket')
|
296
|
+
@bucket.stub!(:[])
|
297
|
+
|
298
|
+
AWS::S3::Bucket.stub!(:find).and_return(@bucket)
|
299
|
+
|
300
|
+
@fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
|
301
|
+
|
302
|
+
@old_stdout, $stdout = $stdout, StringIO.new
|
303
|
+
end
|
304
|
+
|
305
|
+
after(:each) do
|
306
|
+
$stdout = @old_stdout
|
307
|
+
end
|
308
|
+
|
309
|
+
it "deletes the metadata cache file from the S3 service" do
|
310
|
+
AWS::S3::S3Object.should_receive(:delete).with('testprefix_metadata.yml', 'testbucket')
|
311
|
+
@fs.reset_cache
|
312
|
+
end
|
313
|
+
|
314
|
+
end
|
315
|
+
|
316
|
+
describe Awshucks::FileStore, '#bucket (private method)' do
|
317
|
+
|
318
|
+
before(:each) do
|
319
|
+
@connection = mock('connection')
|
320
|
+
AWS::S3::Base.should_receive(:establish_connection!).with(@connection)
|
321
|
+
@fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
|
322
|
+
@old_stdout, $stdout = $stdout, StringIO.new
|
323
|
+
end
|
324
|
+
|
325
|
+
after(:each) do
|
326
|
+
$stdout = @old_stdout
|
327
|
+
end
|
328
|
+
|
329
|
+
it "returns the S3 bucket matching the configured bucket name" do
|
330
|
+
bucket = mock('bucket')
|
331
|
+
AWS::S3::Bucket.should_receive(:find).with('testbucket').and_return(bucket)
|
332
|
+
@fs.send(:bucket).should == bucket
|
333
|
+
end
|
334
|
+
|
335
|
+
it "creates the bucket and returns it if the bucket doesn't already exist" do
|
336
|
+
bucket = mock('bucket')
|
337
|
+
|
338
|
+
raised = false
|
339
|
+
AWS::S3::Bucket.should_receive(:find).twice.with('testbucket').and_return do
|
340
|
+
if raised
|
341
|
+
bucket
|
342
|
+
else
|
343
|
+
raised = true
|
344
|
+
raise AWS::S3::ResponseError.new("The specified bucket does not exist", nil)
|
345
|
+
end
|
346
|
+
end
|
347
|
+
AWS::S3::Bucket.should_receive(:create).with('testbucket').and_return(true)
|
348
|
+
|
349
|
+
@fs.send(:bucket).should == bucket
|
350
|
+
end
|
351
|
+
|
352
|
+
end
|